2025-03-31 Mon 02:15:35:608.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: vhost interface name : vhost0 2025-03-31 Mon 02:15:35:608.650 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: vhost IP Address : 10.0.0.17/24 2025-03-31 Mon 02:15:35:608.669 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: vhost gateway : 0.0.0.0 2025-03-31 Mon 02:15:35:608.675 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Ethernet ports : ens3 2025-03-31 Mon 02:15:35:608.680 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Loopback IP : 0.0.0.0 2025-03-31 Mon 02:15:35:608.686 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Servers : 10.0.0.17:5269 2025-03-31 Mon 02:15:35:608.695 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Authentication : 1 2025-03-31 Mon 02:15:35:608.702 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:15:35:608.707 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:15:35:608.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:15:35:608.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Cluster-Name : 2025-03-31 Mon 02:15:35:608.729 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: DNS Servers : 10.0.0.17:53 2025-03-31 Mon 02:15:35:608.736 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: DNS client port : 8098 2025-03-31 Mon 02:15:35:608.741 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: DNS timeout : 3000 2025-03-31 Mon 02:15:35:608.747 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: DNS max retries : 2 2025-03-31 Mon 02:15:35:608.753 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Dns Authentication : 1 2025-03-31 Mon 02:15:35:608.758 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:15:35:608.763 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:15:35:608.769 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:15:35:608.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TSN Servers : [] 2025-03-31 Mon 02:15:35:608.782 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: COLLECTOR Servers : 10.0.0.17:8086 2025-03-31 Mon 02:15:35:608.787 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Tunnel-Type : MPLSoGRE 2025-03-31 Mon 02:15:35:608.793 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Metadata-Proxy Shared Secret: 774a9b81-45b4-469e-b7cf-04f943bf9586 2025-03-31 Mon 02:15:35:608.836 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Metadata-Proxy Port : 8097 2025-03-31 Mon 02:15:35:608.847 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Metadata-Proxy SSL Flag : 0 2025-03-31 Mon 02:15:35:608.876 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Max Vm Flows : 100 2025-03-31 Mon 02:15:35:608.890 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Linklocal Max System Flows : 16384 2025-03-31 Mon 02:15:35:608.895 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Linklocal Max Vm Flows : 16384 2025-03-31 Mon 02:15:35:608.899 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow cache timeout : 0 2025-03-31 Mon 02:15:35:608.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Stale Interface cleanup timeout : 60 2025-03-31 Mon 02:15:35:608.908 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow thread count : 1 2025-03-31 Mon 02:15:35:608.912 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow latency limit : 0 2025-03-31 Mon 02:15:35:608.916 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow index-mgr sm log count : 0 2025-03-31 Mon 02:15:35:608.921 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow add-tokens : 50 2025-03-31 Mon 02:15:35:608.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow ksync-tokens : 25 2025-03-31 Mon 02:15:35:608.929 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow del-tokens : 16 2025-03-31 Mon 02:15:35:608.933 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow update-tokens : 16 2025-03-31 Mon 02:15:35:608.937 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Pin flow netlink task to CPU: 2025-03-31 Mon 02:15:35:608.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Maximum sessions : 100 2025-03-31 Mon 02:15:35:608.946 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Maximum session aggregates : 8 2025-03-31 Mon 02:15:35:608.950 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Maximum session endpoints : 5 2025-03-31 Mon 02:15:35:608.954 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Fabric SNAT hash table size : 4096 2025-03-31 Mon 02:15:35:608.958 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Flow excluding Router ID in hash :0 2025-03-31 Mon 02:15:35:608.962 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Agent Mode : Vrouter 2025-03-31 Mon 02:15:35:608.967 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Gateway Mode : None 2025-03-31 Mon 02:15:35:608.971 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: DHCP Relay Mode : 0 2025-03-31 Mon 02:15:35:608.976 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance netns cmd : /usr/local/bin/opencontrail-vrouter-netns 2025-03-31 Mon 02:15:35:609.157 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance docker cmd : /usr/local/bin/opencontrail-vrouter-docker 2025-03-31 Mon 02:15:35:609.163 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance workers : 0 2025-03-31 Mon 02:15:35:609.168 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance timeout : 0 2025-03-31 Mon 02:15:35:609.172 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance lb ssl : 2025-03-31 Mon 02:15:35:609.177 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Service instance lbaas auth : 2025-03-31 Mon 02:15:35:609.181 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Bgp as a service port range : 50000-50512 2025-03-31 Mon 02:15:35:609.191 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Services queue limit : 1024 2025-03-31 Mon 02:15:35:609.196 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: BGPAAS max shared sessions for service port : 4 2025-03-31 Mon 02:15:35:609.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:15:35:609.205 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:15:35:609.209 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:15:35:609.213 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Sandesh SSL Enable : 1 2025-03-31 Mon 02:15:35:609.218 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Introspect SSL Enable : 1 2025-03-31 Mon 02:15:35:609.222 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Hypervisor mode : kvm 2025-03-31 Mon 02:15:35:609.444 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Ethernet Port Encap Type : 2025-03-31 Mon 02:15:35:609.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: Platform mode : Vrouter on host linux kernel 2025-03-31 Mon 02:15:35:609.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2025-03-31 00:25:24.263439","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-8423-1.novalocal","build-id":"unknown","build-number":"unknown"}]} src/contrail-common/base/misc_utils.cc 41 2025-03-31 Mon 02:15:35:609.988 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Logging: DISABLED -> ENABLED 2025-03-31 Mon 02:15:35:610.042 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2025-03-31 Mon 02:15:35:769.237 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.274 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2025-03-31 Mon 02:15:35:769.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:15:35:769.287 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:15:35:769.293 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2025-03-31 Mon 02:15:35:769.303 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2025-03-31 Mon 02:15:35:769.307 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2025-03-31 Mon 02:15:35:769.313 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.325 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2025-03-31 Mon 02:15:35:769.329 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:15:35:769.335 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2025-03-31 Mon 02:15:35:769.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:15:35:769.349 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.353 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2025-03-31 Mon 02:15:35:769.359 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2025-03-31 Mon 02:15:35:769.363 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2025-03-31 Mon 02:15:35:769.368 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.373 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2025-03-31 Mon 02:15:35:769.378 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2025-03-31 Mon 02:15:35:769.383 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2025-03-31 Mon 02:15:35:769.388 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2025-03-31 Mon 02:15:35:769.398 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2025-03-31 Mon 02:15:35:769.403 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2025-03-31 Mon 02:15:35:769.409 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2025-03-31 Mon 02:15:35:769.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2025-03-31 Mon 02:15:35:769.421 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:15:35:769.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2025-03-31 Mon 02:15:35:769.431 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2025-03-31 Mon 02:15:35:769.438 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2025-03-31 Mon 02:15:35:769.443 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for routing-instance 2025-03-31 Mon 02:15:35:769.448 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2025-03-31 Mon 02:15:35:769.453 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2025-03-31 Mon 02:15:35:769.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2025-03-31 Mon 02:15:35:769.462 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:15:35:769.466 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:15:35:769.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for security-group 2025-03-31 Mon 02:15:35:769.477 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-group : security-group-access-control-list -> self 2025-03-31 Mon 02:15:35:769.482 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2025-03-31 Mon 02:15:35:769.489 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2025-03-31 Mon 02:15:35:769.496 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for tag 2025-03-31 Mon 02:15:35:769.500 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> self 2025-03-31 Mon 02:15:35:769.506 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:769.510 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2025-03-31 Mon 02:15:35:769.515 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2025-03-31 Mon 02:15:35:769.520 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.525 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:15:35:769.530 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:769.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:769.539 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:15:35:769.548 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:15:35:769.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.560 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-03-31 Mon 02:15:35:769.579 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:15:35:769.586 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2025-03-31 Mon 02:15:35:769.593 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.599 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-03-31 Mon 02:15:35:769.605 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:15:35:769.611 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2025-03-31 Mon 02:15:35:769.618 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.624 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:15:35:769.631 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:769.638 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:769.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:15:35:769.650 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:15:35:769.655 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2025-03-31 Mon 02:15:35:769.665 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:15:35:769.670 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:15:35:769.675 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:15:35:769.683 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2025-03-31 Mon 02:15:35:769.688 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.693 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2025-03-31 Mon 02:15:35:769.697 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2025-03-31 Mon 02:15:35:769.702 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2025-03-31 Mon 02:15:35:769.707 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.713 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.725 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.731 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2025-03-31 Mon 02:15:35:769.736 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2025-03-31 Mon 02:15:35:769.741 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2025-03-31 Mon 02:15:35:769.747 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.751 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.756 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.760 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-03-31 Mon 02:15:35:769.771 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:15:35:769.775 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:15:35:769.780 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:15:35:769.788 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:15:35:769.796 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.801 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.805 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.813 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.818 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-03-31 Mon 02:15:35:769.822 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:15:35:769.827 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:15:35:769.831 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:15:35:769.835 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:15:35:769.841 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.845 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.850 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.854 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.859 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2025-03-31 Mon 02:15:35:769.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2025-03-31 Mon 02:15:35:769.872 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2025-03-31 Mon 02:15:35:769.877 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.881 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.886 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.891 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2025-03-31 Mon 02:15:35:769.898 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.907 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2025-03-31 Mon 02:15:35:769.913 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2025-03-31 Mon 02:15:35:769.917 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2025-03-31 Mon 02:15:35:769.923 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.927 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.932 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.938 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2025-03-31 Mon 02:15:35:769.947 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.957 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.963 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-03-31 Mon 02:15:35:769.968 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:15:35:769.972 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:15:35:769.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:769.983 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:769.988 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2025-03-31 Mon 02:15:35:769.993 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2025-03-31 Mon 02:15:35:769.998 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:15:35:770.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:15:35:770.011 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.018 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:770.024 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.029 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.034 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2025-03-31 Mon 02:15:35:770.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2025-03-31 Mon 02:15:35:770.043 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2025-03-31 Mon 02:15:35:770.048 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2025-03-31 Mon 02:15:35:770.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.057 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:770.061 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.066 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.071 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2025-03-31 Mon 02:15:35:770.076 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2025-03-31 Mon 02:15:35:770.081 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.085 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2025-03-31 Mon 02:15:35:770.092 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2025-03-31 Mon 02:15:35:770.098 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2025-03-31 Mon 02:15:35:770.104 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.108 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2025-03-31 Mon 02:15:35:770.113 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2025-03-31 Mon 02:15:35:770.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2025-03-31 Mon 02:15:35:770.126 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.130 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-03-31 Mon 02:15:35:770.135 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.140 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.145 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.150 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.154 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:15:35:770.159 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-03-31 Mon 02:15:35:770.165 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.172 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-03-31 Mon 02:15:35:770.176 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.181 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.185 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:15:35:770.190 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2025-03-31 Mon 02:15:35:770.196 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2025-03-31 Mon 02:15:35:770.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2025-03-31 Mon 02:15:35:770.209 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2025-03-31 Mon 02:15:35:770.213 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.221 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.226 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.230 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.235 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.241 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:770.250 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.255 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:15:35:770.259 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:770.264 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:770.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> virtual-network-tag 2025-03-31 Mon 02:15:35:770.273 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2025-03-31 Mon 02:15:35:770.278 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.287 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.291 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.296 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.300 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:770.306 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.310 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-03-31 Mon 02:15:35:770.319 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:15:35:770.325 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:15:35:770.330 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-03-31 Mon 02:15:35:770.335 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-03-31 Mon 02:15:35:770.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.352 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.356 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.361 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:770.367 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.371 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-03-31 Mon 02:15:35:770.376 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:15:35:770.380 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:15:35:770.385 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:15:35:770.390 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:15:35:770.397 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-03-31 Mon 02:15:35:770.402 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-03-31 Mon 02:15:35:770.406 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.410 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.422 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.426 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.431 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.435 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:770.441 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.445 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:770.455 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap project : self -> project-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.459 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> project-tag 2025-03-31 Mon 02:15:35:770.475 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2025-03-31 Mon 02:15:35:770.479 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.484 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:15:35:770.488 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.492 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.499 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:770.503 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:770.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.513 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2025-03-31 Mon 02:15:35:770.519 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.523 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.528 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.535 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2025-03-31 Mon 02:15:35:770.542 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:15:35:770.549 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2025-03-31 Mon 02:15:35:770.554 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router : self -> logical-router-interface 2025-03-31 Mon 02:15:35:770.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-router : logical-router-interface -> nil 2025-03-31 Mon 02:15:35:770.564 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for physical-interface 2025-03-31 Mon 02:15:35:770.577 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2025-03-31 Mon 02:15:35:770.585 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:15:35:770.591 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:15:35:770.598 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for physical-interface 2025-03-31 Mon 02:15:35:770.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2025-03-31 Mon 02:15:35:770.614 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2025-03-31 Mon 02:15:35:770.620 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2025-03-31 Mon 02:15:35:770.627 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2025-03-31 Mon 02:15:35:770.634 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2025-03-31 Mon 02:15:35:770.642 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:15:35:770.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2025-03-31 Mon 02:15:35:770.655 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2025-03-31 Mon 02:15:35:770.661 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-03-31 Mon 02:15:35:770.667 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:15:35:770.677 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:15:35:770.684 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:15:35:770.691 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2025-03-31 Mon 02:15:35:770.698 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2025-03-31 Mon 02:15:35:770.705 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2025-03-31 Mon 02:15:35:770.712 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:15:35:770.719 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2025-03-31 Mon 02:15:35:770.725 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2025-03-31 Mon 02:15:35:770.732 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2025-03-31 Mon 02:15:35:770.739 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for physical-router 2025-03-31 Mon 02:15:35:770.745 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2025-03-31 Mon 02:15:35:770.752 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2025-03-31 Mon 02:15:35:770.760 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2025-03-31 Mon 02:15:35:770.769 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for service-health-check 2025-03-31 Mon 02:15:35:770.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-03-31 Mon 02:15:35:770.782 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-03-31 Mon 02:15:35:770.789 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2025-03-31 Mon 02:15:35:770.796 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:770.803 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:15:35:770.810 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:15:35:770.818 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for service-health-check 2025-03-31 Mon 02:15:35:770.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-03-31 Mon 02:15:35:770.839 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-03-31 Mon 02:15:35:770.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2025-03-31 Mon 02:15:35:770.853 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.859 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.872 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2025-03-31 Mon 02:15:35:770.879 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:15:35:770.885 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:15:35:770.892 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:15:35:770.900 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for qos-config 2025-03-31 Mon 02:15:35:770.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2025-03-31 Mon 02:15:35:770.916 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2025-03-31 Mon 02:15:35:770.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for forwarding-class 2025-03-31 Mon 02:15:35:770.933 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2025-03-31 Mon 02:15:35:770.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2025-03-31 Mon 02:15:35:770.948 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2025-03-31 Mon 02:15:35:770.956 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:15:35:770.962 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2025-03-31 Mon 02:15:35:770.977 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2025-03-31 Mon 02:15:35:770.984 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2025-03-31 Mon 02:15:35:770.996 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:15:35:771.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2025-03-31 Mon 02:15:35:771.009 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2025-03-31 Mon 02:15:35:771.016 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2025-03-31 Mon 02:15:35:771.023 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:15:35:771.030 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2025-03-31 Mon 02:15:35:771.037 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2025-03-31 Mon 02:15:35:771.044 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2025-03-31 Mon 02:15:35:771.052 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2025-03-31 Mon 02:15:35:771.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:15:35:771.066 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2025-03-31 Mon 02:15:35:771.074 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2025-03-31 Mon 02:15:35:771.081 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2025-03-31 Mon 02:15:35:771.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2025-03-31 Mon 02:15:35:771.110 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for virtual-router 2025-03-31 Mon 02:15:35:771.117 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-router : virtual-router-sub-cluster -> self 2025-03-31 Mon 02:15:35:771.125 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap sub-cluster : self -> virtual-router-sub-cluster 2025-03-31 Mon 02:15:35:771.131 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap sub-cluster : bgp-router-sub-cluster -> virtual-router-sub-cluster 2025-03-31 Mon 02:15:35:771.138 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-router : self -> bgp-router-sub-cluster 2025-03-31 Mon 02:15:35:771.145 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bgp-router : bgp-router-sub-cluster -> nil 2025-03-31 Mon 02:15:35:771.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for bridge-domain 2025-03-31 Mon 02:15:35:771.167 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-03-31 Mon 02:15:35:771.174 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:15:35:771.180 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2025-03-31 Mon 02:15:35:771.188 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:15:35:771.194 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:15:35:771.201 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for bridge-domain 2025-03-31 Mon 02:15:35:771.207 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-03-31 Mon 02:15:35:771.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:15:35:771.220 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-03-31 Mon 02:15:35:771.228 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for application-policy-set 2025-03-31 Mon 02:15:35:771.234 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2025-03-31 Mon 02:15:35:771.241 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:771.246 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:771.253 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:15:35:771.259 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:15:35:771.267 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:15:35:771.272 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:15:35:771.279 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.292 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.301 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2025-03-31 Mon 02:15:35:771.316 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2025-03-31 Mon 02:15:35:771.323 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:15:35:771.329 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:15:35:771.335 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.341 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.347 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.355 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.361 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2025-03-31 Mon 02:15:35:771.365 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2025-03-31 Mon 02:15:35:771.370 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> address-group-tag 2025-03-31 Mon 02:15:35:771.375 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : address-group-tag -> nil 2025-03-31 Mon 02:15:35:771.380 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:15:35:771.384 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:15:35:771.389 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.397 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.402 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : self -> firewall-rule-tag 2025-03-31 Mon 02:15:35:771.411 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap tag : firewall-rule-tag -> nil 2025-03-31 Mon 02:15:35:771.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:15:35:771.423 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:15:35:771.427 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.436 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:15:35:771.441 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2025-03-31 Mon 02:15:35:954.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.17:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:15:35:959.633 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Current receive sock buffer size is 262144 2025-03-31 Mon 02:15:35:959.732 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915019134720, Pid 1167811]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2025-03-31 Mon 02:15:35:964.826 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Vrouter family is 34 2025-03-31 Mon 02:15:35:965.141 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Flow table size : 20185088 2025-03-31 Mon 02:15:35:965.220 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Flow table size : 161218560 2025-03-31 Mon 02:15:35:965.233 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2025-03-31 Mon 02:15:35:965.250 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2025-03-31 Mon 02:15:35:972.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7f407e000000 2025-03-31 Mon 02:15:35:972.238 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2025-03-31 Mon 02:15:35:972.247 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7f4074600000 2025-03-31 Mon 02:15:35:972.261 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Sending Huge Page configuration to VROUTER 2025-03-31 Mon 02:15:36:026.884 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Huge pages set successfully 2025-03-31 Mon 02:15:36:040.047 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Flow table size : 161218560 2025-03-31 Mon 02:15:36:040.187 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Flow table size : 20185088 2025-03-31 Mon 02:15:36:105.290 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: ROLE : Generator 2025-03-31 Mon 02:15:36:105.318 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: MODULE : contrail-vrouter-agent 2025-03-31 Mon 02:15:36:105.323 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-494-1 2025-03-31 Mon 02:15:36:105.334 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: NODE TYPE : Compute 2025-03-31 Mon 02:15:36:105.338 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: INSTANCE ID : 0 2025-03-31 Mon 02:15:36:105.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: HTTP SERVER PORT : 8085 2025-03-31 Mon 02:15:36:105.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: System Log Send Rate Limit: 0 -> 0 2025-03-31 Mon 02:15:36:120.178 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: HTTP Introspect Init 2025-03-31 Mon 02:15:36:120.388 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:15:36:120.463 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Sandesh Http Server Port: 8085 2025-03-31 Mon 02:15:36:120.517 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1166466.http_port 2025-03-31 Mon 02:15:36:120.524 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: CONNECT TO COLLECTOR: 1 2025-03-31 Mon 02:15:36:120.633 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Idle 2025-03-31 Mon 02:15:36:126.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: No Client: 1743387336120723 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:36:133.502 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvStart in state Idle 2025-03-31 Mon 02:15:36:138.304 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Disconnect 2025-03-31 Mon 02:15:36:138.730 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvCollectorUpdate in state Disconnect 2025-03-31 Mon 02:15:36:139.067 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:15:36:139.151 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:144.861 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2025-03-31 Mon 02:15:36:144.890 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2025-03-31 Mon 02:15:36:144.920 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2025-03-31 Mon 02:15:36:144.926 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2025-03-31 Mon 02:15:36:144.950 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2025-03-31 Mon 02:15:36:144.958 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface veth0ebd6937 flags 69699 2025-03-31 Mon 02:15:36:145.025 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface veth4dff224f flags 69699 2025-03-31 Mon 02:15:36:145.033 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface veth7454db11 flags 69699 2025-03-31 Mon 02:15:36:145.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vetha86214ed flags 69699 2025-03-31 Mon 02:15:36:145.064 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vetha1439fdb flags 69699 2025-03-31 Mon 02:15:36:145.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface veth3b613a14 flags 69699 2025-03-31 Mon 02:15:36:145.094 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vethc9574a7b flags 69699 2025-03-31 Mon 02:15:36:145.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vethb376b920 flags 69699 2025-03-31 Mon 02:15:36:145.124 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vethb28f070c flags 69699 2025-03-31 Mon 02:15:36:145.150 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface veth7646ab2c flags 69699 2025-03-31 Mon 02:15:36:145.155 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vethaeed3a20 flags 69699 2025-03-31 Mon 02:15:36:145.177 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vethc08d8cac flags 69699 2025-03-31 Mon 02:15:36:145.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2025-03-31 Mon 02:15:36:145.205 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2025-03-31 Mon 02:15:36:145.210 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2025-03-31 Mon 02:15:36:145.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2025-03-31 Mon 02:15:36:145.221 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2025-03-31 Mon 02:15:36:145.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2025-03-31 Mon 02:15:36:145.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface lo 2025-03-31 Mon 02:15:36:145.292 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-03-31 Mon 02:15:36:145.300 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2025-03-31 Mon 02:15:36:145.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface docker0 2025-03-31 Mon 02:15:36:145.317 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-03-31 Mon 02:15:36:145.331 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface lo 2025-03-31 Mon 02:15:36:145.340 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-03-31 Mon 02:15:36:145.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface ftun0 2025-03-31 Mon 02:15:36:145.356 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-03-31 Mon 02:15:36:145.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink address message RTM_NEWADDR for interface pkt0 2025-03-31 Mon 02:15:36:145.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.440 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.460 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.484 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.499 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 10.94.185.0/24 Interface lxdbr0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.518 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.527 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2025-03-31 Mon 02:15:36:145.532 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.536 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:15:36:145.540 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.544 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:15:36:145.549 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.553 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.557 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:15:36:145.561 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.565 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:15:36:145.577 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:15:36:145.582 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:15:36:145.596 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.600 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.605 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.613 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:15:36:145.617 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:15:36:145.621 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:15:36:145.626 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:15:36:145.633 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:15:36:145.637 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:15:36:145.641 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:15:36:145.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:15:36:145.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:15:36:145.653 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:15:36:145.658 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:15:36:145.662 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:15:36:145.666 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:15:36:150.687 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138064 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-03-31 Mon 02:15:36:150.726 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-03-31 Mon 02:15:36:150.750 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:150.765 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138116 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-03-31 Mon 02:15:36:150.770 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-03-31 Mon 02:15:36:150.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:150.797 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138292 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:15:36:150.804 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:36:150.814 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:150.914 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138716 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:36:150.932 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:36:150.943 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:151.006 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138772 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:36:151.017 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:36:151.024 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:151.042 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336138789 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:15:36:151.047 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:36:151.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:151.110 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336139139 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:36:151.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:36:164.198 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:15:36:170.290 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Router ID configured. Connection to controller initiated 2025-03-31 Mon 02:15:36:215.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:215.602 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336215152 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:215.617 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:218.985 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server [fe80::f816:3eff:fec0:9c69%vhost0]:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:15:36:220.246 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:220.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336215742 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:15:36:220.299 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:36:220.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:220.320 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336216088 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:220.326 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:220.333 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:220.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336216110 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, XMPP:dns-server:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:15:36:220.354 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:36:264.273 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:265.484 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387336264225 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:265.507 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:265.856 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:265.883 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387336265836 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 42021 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:265.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:293.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:42021::10.0.0.17:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-03-31 Mon 02:15:36:293.446 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-03-31 Mon 02:15:36:301.193 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:301.250 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387336301143 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 42021 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:301.259 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:301.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:301.279 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387336301195 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:301.284 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.058 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:33107::10.0.0.17:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-03-31 Mon 02:15:36:304.090 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-03-31 Mon 02:15:36:304.400 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.437 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336304145 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 33107 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.449 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.456 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336304240 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 33107 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.481 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.597 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:15:36:304.624 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336304494 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.640 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.654 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.668 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336304585 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.674 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.750 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.792 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336304731 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.805 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:304.861 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:304.875 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:15:36:304.876 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387336304854 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:304.893 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:307.601 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:307.607 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:15:36:307.715 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:15:36:307.856 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336307564 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:307.867 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:307.880 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:307.888 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336307698 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:307.893 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:316.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-03-31 Mon 02:15:36:319.254 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-03-31 Mon 02:15:36:319.283 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:319.611 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336319262 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:319.628 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:319.638 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:319.648 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387336319407 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:319.653 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:319.947 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:319.987 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336319926 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:319.997 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:320.008 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:320.037 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387336319972 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:15:36:320.044 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:36:320.412 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:320.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387336320400 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:15:36:320.447 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:15:36:320.454 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:36:320.480 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387336320438 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:15:36:320.488 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:37:139.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:37:139.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:37:139.279 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:15:37:139.390 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Idle 2025-03-31 Mon 02:15:37:139.482 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:37:139.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387337139310 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:15:37:139.517 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:37:139.524 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:15:37:139.532 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:37:139.679 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387337139473 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 29, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:37:139.699 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:37:139.731 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:15:38:139.639 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:15:38:140.454 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:15:38:140.691 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:38:140.727 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387338139964 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:15:38:140.735 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:38:140.743 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:38:140.882 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387338140533 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 31, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:38:140.899 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:39:075.492 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:39:075.560 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387339075439 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:39:075.580 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:39:082.013 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:39:082.061 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387339081983 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:39:082.078 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:39:568.949 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:39:569.030 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387339567357 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric acl_uuid = 57fa88d5-1162-43c2-ba82-a5761f77fd36 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:15:39:569.042 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:15:39:674.126 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:39:674.787 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387339636883 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric acl_uuid = 57fa88d5-1162-43c2-ba82-a5761f77fd36 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:15:39:674.812 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:15:39:848.048 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:39:848.096 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387339848027 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric acl_uuid = 57fa88d5-1162-43c2-ba82-a5761f77fd36 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:15:39:848.105 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:15:40:140.554 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:40:140.617 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:40:140.626 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:15:40:141.135 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 335544539 category: asio.ssl message: short read src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:15:40:141.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Idle 2025-03-31 Mon 02:15:40:141.271 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:40:141.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387340140655 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:40:141.304 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:40:141.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:15:40:141.347 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:40:141.481 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387340141260 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 38, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:40:141.497 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:42:145.367 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:15:42:145.696 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:15:42:145.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:42:145.827 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387342145442 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:42:145.834 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:42:145.842 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:42:145.977 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387342145786 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 40, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:42:145.996 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:46:147.784 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:46:155.353 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:46:155.384 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:15:46:161.695 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Idle 2025-03-31 Mon 02:15:46:161.855 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:46:161.905 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387346161611 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:46:161.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:46:161.938 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:15:46:161.952 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:46:162.185 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387346161838 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 42, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:46:162.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:46:163.896 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:15:50:161.822 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:15:50:211.523 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:15:50:211.717 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:50:211.764 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387350211266 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:50:211.773 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:50:211.783 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:15:50:211.922 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387350211694 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 44, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:50:211.945 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:15:58:211.724 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:58:211.817 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:15:58:211.854 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:15:58:211.987 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Idle 2025-03-31 Mon 02:15:58:212.005 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:15:58:212.066 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:58:212.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387358211891 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:15:58:212.111 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:15:58:212.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:15:58:212.158 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:15:58:212.299 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387358212056 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 46, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:15:58:212.316 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:02:588.899 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:48098 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:16:02:633.871 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:48098 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:16:05:643.826 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:643.936 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387365643777 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric total_acl_rules = 4 interface_list= [ [ (*_iter66) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0, ] ] virtualmachine_list= [ [ ] ] acl = default-domain:default-project:ip-fabric:ip-fabric mirror_acl = ] 2025-03-31 Mon 02:16:05:643.947 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:16:05:650.208 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:650.273 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387365650159 [SYS_NOTICE]: UveVMInterfaceAgentTrace: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 ip_address = 10.0.0.17 virtual_network = default-domain:default-project:ip-fabric mac_address = fa:16:3e:c0:9c:69 label = 17 active = 1 vm_name = l2_active = 1 uuid = cca6d80d-e05d-4411-8298-d2b2a4db205d ip6_address = :: ip6_active = 1 floating_ips= [ [ ] ] vm_uuid = ip4_active = 1 is_health_check_active = 1 health_check_instance_list= [ [ ] ] admin_state = 1 fixed_ip4_list= [ [ (*_iter26) = 10.0.0.17, ] ] fixed_ip6_list= [ [ ] ] vn_uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f tx_vlan = 65535 rx_vlan = 65535 parent_interface = ens3 vhostuser_mode = 0 port_mirror_enabled = 0 ] 2025-03-31 Mon 02:16:05:650.284 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVMInterfaceAgentTrace 2025-03-31 Mon 02:16:05:650.293 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:650.304 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387365650200 [SYS_NOTICE]: UVEVMITags: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 app = tier = site = deployment = labels= [ [ ] ] custom_tags= [ [ ] ] ] 2025-03-31 Mon 02:16:05:650.310 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UVEVMITags 2025-03-31 Mon 02:16:05:651.479 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:651.577 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387365651455 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric associated_fip_count = 0 in_bandwidth_usage = 0 out_bandwidth_usage = 0 vrf_stats_list= [ [ [ name = default-domain:default-project:ip-fabric:ip-fabric diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:16:05:651.597 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:16:05:651.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:651.640 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387365651471 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2025-03-31 Mon 02:16:05:651.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:16:05:651.713 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:651.847 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387365651673 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric port_bucket_bmap= [ udp_sport_bitmap= [ [ (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, ] ] udp_dport_bitmap= [ [ (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, ] ] tcp_sport_bitmap= [ [ (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, ] ] tcp_dport_bitmap= [ [ (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, ] ] ] raw_if_stats= [ in_pkts = 120 in_bytes = 9634 out_pkts = 123 out_bytes = 9596 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 0 out_bw_usage = 0 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:16:05:651.864 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:16:05:651.872 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:651.882 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387365651712 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:16:05:651.888 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:16:05:652.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:652.871 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387365652744 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 connected_networks= [ [ (*_iter142) = default-domain:default-project:ip-fabric, ] ] vn_count = 1 ] 2025-03-31 Mon 02:16:05:652.882 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:16:05:652.892 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:05:652.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387365652803 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 interface_list= [ [ (*_iter143) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0, ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ ] ] vmi_count= [ active = 1 ] down_interface_count = 0 ] 2025-03-31 Mon 02:16:05:652.912 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:16:06:126.895 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:06:126.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387366126711 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 3 dequeues = 3 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 57 dequeues = 57 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 70 dequeues = 70 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387358211985 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387365652891 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:16:06:127.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:06:127.011 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:06:127.217 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387366126836 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 57, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:16:06:127.234 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:06:146.094 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:06:146.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387366146032 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 xmpp_peer_list= [ [ [ ip = 10.0.0.17 status = 1 setup_time = 1743387336320444 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.17, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2025-03-31 00:25:24.263439","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-8423-1.novalocal","build-id":"unknown","build-number":"unknown"}]} phy_if= [ [ [ name = ens3 mac_address = fa:16:3e:c0:9c:69 ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:c0:9c:69 ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_DEBUG log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.17, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.17 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = ens3, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.17 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.17:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.17:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-juju-os-494-1 flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.17:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2025-03-31 00:22:50.809725", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-8423-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2025-03-31 Mon 02:16:06:146.219 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:16:06:146.229 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:06:146.483 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387366146104 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 120 in_bytes = 9634 out_tpkts = 123 out_bytes = 9596 exception_packets = 21 exception_packets_dropped = 11 exception_packets_allowed = 10 total_flows = 0 aged_flows = 0 uptime = 1743387335649950 phy_if_info= [ [ _iter254->first = ens3 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 120 in_bytes = 9634 out_pkts = 123 out_bytes = 9596 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 98 in_bytes = 8546 out_pkts = 123 out_bytes = 9760 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:16:06:146.508 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:16:06:146.519 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:06:146.558 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387366146161 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:16:06:146.568 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:16:06:212.160 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:16:06:212.478 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:16:06:212.597 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:16:06:212.636 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387366212249 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:16:06:212.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:16:06:212.659 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:16:06:212.864 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387366212583 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 62, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:16:06:212.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:13:161.249 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40288 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:16:13:209.871 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40288 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:16:20:959.587 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40290 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:16:22:212.634 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:16:22:212.690 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:16:22:212.699 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:16:22:212.865 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Idle 2025-03-31 Mon 02:16:22:212.912 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 335544539 category: asio.ssl message: short read src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:16:22:213.052 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:22:213.091 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387382212744 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:16:22:213.099 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:16:22:213.108 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:16:22:213.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:22:213.372 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387382213032 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 23, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 64, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:16:22:213.406 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:35:653.069 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:35:653.134 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387395653035 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:16:35:653.145 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:16:35:653.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:35:653.489 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387395653370 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 145 in_bytes = 11057 out_pkts = 148 out_bytes = 10911 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 379 out_bw_usage = 350 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:16:35:653.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:16:35:653.525 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:35:653.535 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387395653378 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:16:35:653.541 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:16:36:127.082 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:127.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387396126942 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 69 dequeues = 69 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 85 dequeues = 85 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387382212862 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387395653523 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:16:36:127.169 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:36:127.177 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:127.394 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387396127051 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 23, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 69, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:16:36:127.415 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:16:36:147.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:147.148 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387396146957 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 2286366254 buffers = 32576 cached = 25466304 node_type = ] meminfo= [ virt = 32576 peakvirt = 2272332184 res = 32576 ] cpuload= [ one_min_avg = 0.9025 five_min_avg = 0.65 fifteen_min_avg = 0.39 ] cpu_share = 0.485961 ] ] 2025-03-31 Mon 02:16:36:147.172 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:16:36:147.186 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:147.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387396147003 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32576 cpu_share = 0.485961 used_sys_mem = 4269468416 one_min_cpuload = 0.9025 mem_res = 32576 ], ] ] ] 2025-03-31 Mon 02:16:36:147.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:16:36:147.247 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:147.394 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387396147032 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 145 in_bytes = 11057 out_tpkts = 148 out_bytes = 10911 exception_packets = 23 exception_packets_dropped = 12 exception_packets_allowed = 11 raw_vhost_stats= [ name = vhost0 in_pkts = 145 in_bytes = 11057 out_pkts = 148 out_bytes = 10911 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 108 in_bytes = 9231 out_pkts = 148 out_bytes = 11183 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:16:36:147.408 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:16:36:147.415 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:16:36:147.448 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387396147059 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:16:36:147.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:16:38:213.054 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:16:38:213.385 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:16:38:213.510 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:16:38:213.540 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387398213160 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:16:38:213.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:16:38:213.563 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:16:38:213.827 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387398213494 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 23, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 75, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:16:38:213.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:00:775.983 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41928 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:17:00:817.837 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41928 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:17:05:653.129 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:05:653.194 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387425653096 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:17:05:653.205 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:17:05:653.310 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:05:653.423 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387425653283 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 170 in_bytes = 12452 out_pkts = 175 out_bytes = 12338 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 384 out_bw_usage = 393 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:17:05:653.451 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:17:05:653.460 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:05:653.471 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387425653293 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:17:05:653.477 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:17:06:127.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:06:127.498 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387426127227 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 80 dequeues = 80 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 97 dequeues = 97 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387398213166 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387425653459 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:17:06:127.513 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:06:127.522 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:06:127.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387426127337 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 80, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:17:06:127.793 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:06:147.298 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:06:147.490 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387426147263 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 170 in_bytes = 12452 out_tpkts = 175 out_bytes = 12338 exception_packets = 24 exception_packets_allowed = 12 raw_vhost_stats= [ name = vhost0 in_pkts = 170 in_bytes = 12452 out_pkts = 175 out_bytes = 12338 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] total_in_bandwidth_utilization = 1.06794e-15 total_out_bandwidth_utilization = 2.03288e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 197, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 375, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 120 in_bytes = 10028 out_pkts = 173 out_bytes = 12578 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:17:06:147.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:17:06:147.528 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:06:147.571 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387426147316 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:17:06:147.587 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:17:08:213.556 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:17:08:213.657 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:17:08:213.694 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:17:08:213.824 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Idle 2025-03-31 Mon 02:17:08:213.855 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:17:08:213.943 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:08:213.972 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387428213726 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:17:08:213.995 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:17:08:214.006 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:17:08:214.048 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:08:214.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387428213918 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 15 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 26, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 84, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:17:08:214.290 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:21:862.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41678 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:17:35:653.880 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:35:653.940 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387455653841 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:17:35:653.950 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:17:35:654.065 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:35:654.159 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387455654039 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 190 in_bytes = 13500 out_pkts = 194 out_bytes = 13208 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 279 out_bw_usage = 232 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:17:35:654.186 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:17:35:654.194 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:35:654.205 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387455654048 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:17:35:654.211 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:17:36:127.627 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:127.734 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387456127478 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 89 dequeues = 89 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 108 dequeues = 108 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387428213822 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387455654193 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:17:36:127.748 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:36:127.756 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:127.974 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387456127596 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 89, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:17:36:127.999 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:17:36:147.504 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:147.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387456147468 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 2286366254 buffers = 32576 cached = 25466304 node_type = ] meminfo= [ virt = 32576 peakvirt = 2272332184 res = 32576 ] cpuload= [ one_min_avg = 0.47875 five_min_avg = 0.575 fifteen_min_avg = 0.38 ] cpu_share = 0.325 ] ] 2025-03-31 Mon 02:17:36:147.624 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:17:36:147.636 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:147.653 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387456147486 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32576 cpu_share = 0.325 used_sys_mem = 4269468416 one_min_cpuload = 0.47875 mem_res = 32576 ], ] ] ] 2025-03-31 Mon 02:17:36:147.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:17:36:147.666 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:147.827 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387456147516 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 190 in_bytes = 13500 out_tpkts = 194 out_bytes = 13208 exception_packets = 25 exception_packets_dropped = 13 raw_vhost_stats= [ name = vhost0 in_pkts = 190 in_bytes = 13500 out_pkts = 194 out_bytes = 13208 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 124 in_bytes = 10268 out_pkts = 193 out_bytes = 13626 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:17:36:147.851 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:17:36:147.861 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:17:36:147.888 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387456147551 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:17:36:147.894 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:17:38:213.966 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:17:38:214.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:17:38:214.322 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:38:214.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387458214043 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:17:38:214.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:17:38:214.372 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:17:38:214.605 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: SANDESH: Send FAILED: 1743387458214300 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 95, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:17:38:214.627 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914993944320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:05:653.521 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:05:653.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387485653483 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:18:05:653.618 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:18:05:653.689 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:05:653.791 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387485653648 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 209 in_bytes = 14478 out_pkts = 214 out_bytes = 14148 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 269 out_bw_usage = 259 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:18:05:653.817 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:18:05:653.826 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:05:653.837 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387485653671 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:18:05:653.844 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:18:06:127.919 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:06:128.063 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387486127762 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 100 dequeues = 100 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 120 dequeues = 120 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387458214050 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387485653824 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:18:06:128.080 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:06:128.090 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:06:128.381 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387486127874 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 100, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:18:06:128.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:06:147.844 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:06:148.051 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387486147804 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 209 in_bytes = 14478 out_tpkts = 214 out_bytes = 14148 raw_vhost_stats= [ name = vhost0 in_pkts = 209 in_bytes = 14478 out_pkts = 214 out_bytes = 14148 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] total_in_bandwidth_utilization = 3.95734e-16 total_out_bandwidth_utilization = 1.46367e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 73, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 270, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 129 in_bytes = 10578 out_pkts = 212 out_bytes = 14604 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:18:06:148.073 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:18:06:148.083 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:06:148.110 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387486147860 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:18:06:148.116 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:18:08:214.359 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:18:08:214.448 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:18:08:214.484 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:18:08:214.618 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Idle 2025-03-31 Mon 02:18:08:214.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-03-31 Mon 02:18:08:214.727 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:08:214.752 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387488214516 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:18:08:214.769 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:18:08:214.778 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:18:08:214.814 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:08:215.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387488214711 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 104, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:18:08:215.076 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:22:786.331 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:59606 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:18:35:653.746 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:35:653.798 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: SANDESH: Send FAILED: 1743387515653713 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:18:35:653.809 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139914998142720, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:18:35:653.878 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:35:653.983 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387515653854 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 227 in_bytes = 15354 out_pkts = 233 out_bytes = 15022 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 233 out_bw_usage = 233 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:18:35:654.010 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:18:35:654.020 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:35:654.031 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387515653862 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:18:35:654.040 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:18:36:128.128 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:128.203 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387516127992 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 109 dequeues = 109 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 131 dequeues = 131 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387488214616 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387515654019 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:18:36:128.215 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:36:128.223 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:128.449 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387516128103 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 109, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:18:36:129.079 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:18:36:149.937 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:150.033 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387516149899 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 2286366254 buffers = 32576 cached = 25466304 node_type = ] meminfo= [ virt = 32576 peakvirt = 2272332184 res = 32576 ] cpuload= [ one_min_avg = 0.31625 five_min_avg = 0.5125 fifteen_min_avg = 0.37125 ] cpu_share = 0.245833 ] ] 2025-03-31 Mon 02:18:36:150.043 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:18:36:150.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:150.068 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387516149920 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32576 cpu_share = 0.245833 used_sys_mem = 4269468416 one_min_cpuload = 0.31625 mem_res = 32576 ], ] ] ] 2025-03-31 Mon 02:18:36:150.076 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:18:36:150.083 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:150.211 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387516149948 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 227 in_bytes = 15354 out_tpkts = 233 out_bytes = 15022 raw_vhost_stats= [ name = vhost0 in_pkts = 227 in_bytes = 15354 out_pkts = 233 out_bytes = 15022 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 133 in_bytes = 10822 out_pkts = 230 out_bytes = 15480 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:18:36:150.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:18:36:150.234 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:18:36:150.262 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: SANDESH: Send FAILED: 1743387516149982 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:18:36:150.268 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:18:38:214.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:18:38:215.131 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:18:38:215.253 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:38:215.292 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387518214866 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:18:38:215.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:18:38:215.317 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:18:38:215.538 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: SANDESH: Send FAILED: 1743387518215237 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 115, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:18:38:215.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915010737920, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:05:653.999 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:05:654.062 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387545653965 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:19:05:654.073 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:19:05:654.160 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:05:654.291 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387545654133 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 250 in_bytes = 16665 out_pkts = 256 out_bytes = 16225 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 349 out_bw_usage = 320 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:19:05:654.322 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:19:05:654.332 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:05:654.344 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: SANDESH: Send FAILED: 1743387545654138 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:19:05:654.351 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915023333120, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:19:06:128.403 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:06:128.540 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387546128258 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 120 dequeues = 120 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 143 dequeues = 143 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387518214876 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387545654331 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:19:06:128.558 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:06:128.574 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:06:128.838 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387546128358 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 120, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:06:128.870 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:06:150.256 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:06:150.459 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387546150216 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 250 in_bytes = 16665 out_tpkts = 256 out_bytes = 16225 raw_vhost_stats= [ name = vhost0 in_pkts = 250 in_bytes = 16665 out_pkts = 256 out_bytes = 16225 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ] total_in_bandwidth_utilization = 5.85469e-16 total_out_bandwidth_utilization = 1.57751e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 108, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 291, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 141 in_bytes = 11395 out_pkts = 253 out_bytes = 16791 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:19:06:150.488 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:19:06:150.499 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:06:150.526 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: SANDESH: Send FAILED: 1743387546150290 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:19:06:150.533 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915014936320, Pid 1167811]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:19:08:215.350 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915006539520, Pid 1167811]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:08:215.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:08:215.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:19:08:215.656 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Idle 2025-03-31 Mon 02:19:08:215.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:08:215.803 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387548215566 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:08:215.810 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:08:215.819 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:19:08:215.854 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:08:216.076 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: SANDESH: Send FAILED: 1743387548215750 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1167811 http_port = 8085 start_time = 1743387336120644 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 124, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:08:216.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915002341120, Pid 1167811]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:10:520.157 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:39990 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:19:10:561.959 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:39990 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:19:19:296.054 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:45876 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:19:19:337.867 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:45876 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:19:23:770.068 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:50856 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:19:30:884.289 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:50866 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:19:30:929.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 139915172444480, Pid 1167811]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:50866 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:19:40:539.953 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: vhost interface name : vhost0 2025-03-31 Mon 02:19:40:540.082 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: vhost IP Address : 10.0.0.17/24 2025-03-31 Mon 02:19:40:540.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: vhost gateway : 0.0.0.0 2025-03-31 Mon 02:19:40:540.100 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Ethernet ports : ens3 2025-03-31 Mon 02:19:40:540.108 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Loopback IP : 0.0.0.0 2025-03-31 Mon 02:19:40:540.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Servers : 10.0.0.17:5269 2025-03-31 Mon 02:19:40:540.123 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Authentication : 1 2025-03-31 Mon 02:19:40:540.137 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:19:40:540.144 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:19:40:540.150 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:19:40:540.157 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Cluster-Name : 2025-03-31 Mon 02:19:40:540.164 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: DNS Servers : 10.0.0.17:53 2025-03-31 Mon 02:19:40:540.170 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: DNS client port : 8098 2025-03-31 Mon 02:19:40:540.175 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: DNS timeout : 3000 2025-03-31 Mon 02:19:40:540.181 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: DNS max retries : 2 2025-03-31 Mon 02:19:40:540.188 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Dns Authentication : 1 2025-03-31 Mon 02:19:40:540.194 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:19:40:540.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:19:40:540.206 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:19:40:540.213 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TSN Servers : [] 2025-03-31 Mon 02:19:40:540.219 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: COLLECTOR Servers : 10.0.0.17:8086 2025-03-31 Mon 02:19:40:540.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Tunnel-Type : MPLSoGRE 2025-03-31 Mon 02:19:40:540.231 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Metadata-Proxy Shared Secret: 774a9b81-45b4-469e-b7cf-04f943bf9586 2025-03-31 Mon 02:19:40:540.237 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Metadata-Proxy Port : 8097 2025-03-31 Mon 02:19:40:540.243 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Metadata-Proxy SSL Flag : 0 2025-03-31 Mon 02:19:40:540.266 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Max Vm Flows : 100 2025-03-31 Mon 02:19:40:540.274 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Linklocal Max System Flows : 16384 2025-03-31 Mon 02:19:40:540.280 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Linklocal Max Vm Flows : 16384 2025-03-31 Mon 02:19:40:540.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow cache timeout : 0 2025-03-31 Mon 02:19:40:540.291 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Stale Interface cleanup timeout : 60 2025-03-31 Mon 02:19:40:540.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow thread count : 1 2025-03-31 Mon 02:19:40:540.303 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow latency limit : 0 2025-03-31 Mon 02:19:40:540.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow index-mgr sm log count : 0 2025-03-31 Mon 02:19:40:540.321 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow add-tokens : 50 2025-03-31 Mon 02:19:40:540.327 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow ksync-tokens : 25 2025-03-31 Mon 02:19:40:540.333 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow del-tokens : 16 2025-03-31 Mon 02:19:40:540.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow update-tokens : 16 2025-03-31 Mon 02:19:40:540.345 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Pin flow netlink task to CPU: 2025-03-31 Mon 02:19:40:540.352 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Maximum sessions : 100 2025-03-31 Mon 02:19:40:540.358 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Maximum session aggregates : 8 2025-03-31 Mon 02:19:40:540.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Maximum session endpoints : 5 2025-03-31 Mon 02:19:40:540.371 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Fabric SNAT hash table size : 4096 2025-03-31 Mon 02:19:40:540.377 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Flow excluding Router ID in hash :0 2025-03-31 Mon 02:19:40:540.384 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Agent Mode : Vrouter 2025-03-31 Mon 02:19:40:540.390 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Gateway Mode : None 2025-03-31 Mon 02:19:40:540.396 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: DHCP Relay Mode : 0 2025-03-31 Mon 02:19:40:540.402 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance netns cmd : /usr/local/bin/opencontrail-vrouter-netns 2025-03-31 Mon 02:19:40:540.409 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance docker cmd : /usr/local/bin/opencontrail-vrouter-docker 2025-03-31 Mon 02:19:40:540.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance workers : 0 2025-03-31 Mon 02:19:40:540.422 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance timeout : 0 2025-03-31 Mon 02:19:40:540.429 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance lb ssl : 2025-03-31 Mon 02:19:40:540.435 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Service instance lbaas auth : 2025-03-31 Mon 02:19:40:540.441 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Bgp as a service port range : 50000-50512 2025-03-31 Mon 02:19:40:540.447 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Services queue limit : 1024 2025-03-31 Mon 02:19:40:540.453 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: BGPAAS max shared sessions for service port : 4 2025-03-31 Mon 02:19:40:540.459 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2025-03-31 Mon 02:19:40:540.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2025-03-31 Mon 02:19:40:540.471 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2025-03-31 Mon 02:19:40:540.482 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Sandesh SSL Enable : 1 2025-03-31 Mon 02:19:40:540.487 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Introspect SSL Enable : 1 2025-03-31 Mon 02:19:40:540.493 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Hypervisor mode : kvm 2025-03-31 Mon 02:19:40:540.646 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Ethernet Port Encap Type : 2025-03-31 Mon 02:19:40:540.659 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: Platform mode : Vrouter on host linux kernel 2025-03-31 Mon 02:19:40:540.895 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Logging: DISABLED -> ENABLED 2025-03-31 Mon 02:19:40:540.938 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2025-03-31 Mon 02:19:40:553.098 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.143 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2025-03-31 Mon 02:19:40:553.152 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:19:40:553.160 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:19:40:553.167 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.174 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2025-03-31 Mon 02:19:40:553.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2025-03-31 Mon 02:19:40:553.189 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2025-03-31 Mon 02:19:40:553.197 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2025-03-31 Mon 02:19:40:553.210 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:19:40:553.217 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2025-03-31 Mon 02:19:40:553.224 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:19:40:553.231 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.237 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2025-03-31 Mon 02:19:40:553.245 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2025-03-31 Mon 02:19:40:553.261 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2025-03-31 Mon 02:19:40:553.268 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.275 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2025-03-31 Mon 02:19:40:553.283 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2025-03-31 Mon 02:19:40:553.289 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2025-03-31 Mon 02:19:40:553.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.303 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2025-03-31 Mon 02:19:40:553.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2025-03-31 Mon 02:19:40:553.319 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2025-03-31 Mon 02:19:40:553.327 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2025-03-31 Mon 02:19:40:553.335 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2025-03-31 Mon 02:19:40:553.342 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-network 2025-03-31 Mon 02:19:40:553.349 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2025-03-31 Mon 02:19:40:553.356 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2025-03-31 Mon 02:19:40:553.363 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2025-03-31 Mon 02:19:40:553.370 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for routing-instance 2025-03-31 Mon 02:19:40:553.377 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2025-03-31 Mon 02:19:40:566.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2025-03-31 Mon 02:19:40:566.414 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2025-03-31 Mon 02:19:40:566.424 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:19:40:566.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:19:40:566.464 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for security-group 2025-03-31 Mon 02:19:40:566.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-group : security-group-access-control-list -> self 2025-03-31 Mon 02:19:40:566.481 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2025-03-31 Mon 02:19:40:566.488 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2025-03-31 Mon 02:19:40:566.500 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for tag 2025-03-31 Mon 02:19:40:566.507 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> self 2025-03-31 Mon 02:19:40:566.518 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:566.525 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2025-03-31 Mon 02:19:40:566.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2025-03-31 Mon 02:19:40:566.543 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:566.550 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:19:40:566.558 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:566.566 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:566.574 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:19:40:566.595 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:19:40:570.403 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.415 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-03-31 Mon 02:19:40:570.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:19:40:570.433 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2025-03-31 Mon 02:19:40:570.442 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.449 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-03-31 Mon 02:19:40:570.472 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:19:40:570.481 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2025-03-31 Mon 02:19:40:570.490 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.497 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:19:40:570.505 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:570.514 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:570.522 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:19:40:570.529 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:19:40:570.538 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.545 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2025-03-31 Mon 02:19:40:570.558 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:19:40:570.566 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:19:40:570.575 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2025-03-31 Mon 02:19:40:570.598 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2025-03-31 Mon 02:19:40:570.608 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.615 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2025-03-31 Mon 02:19:40:570.622 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2025-03-31 Mon 02:19:40:570.630 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2025-03-31 Mon 02:19:40:570.639 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:570.647 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:570.663 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:570.670 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:570.678 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2025-03-31 Mon 02:19:40:570.686 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2025-03-31 Mon 02:19:40:571.069 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2025-03-31 Mon 02:19:40:571.085 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:571.104 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.112 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.121 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-03-31 Mon 02:19:40:571.130 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:19:40:571.139 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:19:40:571.146 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-03-31 Mon 02:19:40:571.154 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-03-31 Mon 02:19:40:571.162 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.172 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:571.179 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.187 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.194 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-03-31 Mon 02:19:40:571.201 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:19:40:571.216 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2025-03-31 Mon 02:19:40:571.223 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:19:40:571.230 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:19:40:571.240 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.247 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:571.254 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.262 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.270 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2025-03-31 Mon 02:19:40:571.278 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2025-03-31 Mon 02:19:40:571.286 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2025-03-31 Mon 02:19:40:571.295 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.302 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:571.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.319 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2025-03-31 Mon 02:19:40:571.328 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.345 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2025-03-31 Mon 02:19:40:571.354 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2025-03-31 Mon 02:19:40:571.362 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2025-03-31 Mon 02:19:40:571.371 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.378 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:571.387 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2025-03-31 Mon 02:19:40:571.399 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2025-03-31 Mon 02:19:40:571.409 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:571.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:577.560 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2025-03-31 Mon 02:19:40:577.624 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-03-31 Mon 02:19:40:577.636 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:19:40:577.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:19:40:579.061 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:579.106 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.115 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2025-03-31 Mon 02:19:40:579.124 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:19:40:579.131 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:19:40:579.142 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.150 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:579.158 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.166 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.175 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2025-03-31 Mon 02:19:40:579.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2025-03-31 Mon 02:19:40:579.191 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2025-03-31 Mon 02:19:40:579.213 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2025-03-31 Mon 02:19:40:579.222 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.229 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:579.236 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.244 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.253 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2025-03-31 Mon 02:19:40:579.261 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2025-03-31 Mon 02:19:40:579.270 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.277 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2025-03-31 Mon 02:19:40:579.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2025-03-31 Mon 02:19:40:579.292 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2025-03-31 Mon 02:19:40:579.301 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2025-03-31 Mon 02:19:40:579.316 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2025-03-31 Mon 02:19:40:579.323 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2025-03-31 Mon 02:19:40:579.333 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-03-31 Mon 02:19:40:579.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.355 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.372 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.385 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:19:40:579.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-03-31 Mon 02:19:40:579.402 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.409 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-03-31 Mon 02:19:40:579.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.423 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.430 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-03-31 Mon 02:19:40:579.438 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2025-03-31 Mon 02:19:40:579.448 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.455 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2025-03-31 Mon 02:19:40:579.463 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2025-03-31 Mon 02:19:40:579.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2025-03-31 Mon 02:19:40:579.478 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.485 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.494 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.502 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.510 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.518 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:579.528 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.536 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-03-31 Mon 02:19:40:579.547 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:579.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:579.563 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> virtual-network-tag 2025-03-31 Mon 02:19:40:579.570 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2025-03-31 Mon 02:19:40:579.577 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.597 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.604 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.611 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.619 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.626 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:579.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.643 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-03-31 Mon 02:19:40:579.651 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:19:40:579.659 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:19:40:579.667 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-03-31 Mon 02:19:40:579.675 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-03-31 Mon 02:19:40:579.682 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.689 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.696 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.707 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:579.731 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.739 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-03-31 Mon 02:19:40:579.746 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:19:40:579.753 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2025-03-31 Mon 02:19:40:579.761 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:19:40:579.767 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-03-31 Mon 02:19:40:579.774 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-03-31 Mon 02:19:40:579.781 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-03-31 Mon 02:19:40:579.788 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.801 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.808 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.815 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.860 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:579.875 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.883 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:579.904 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap project : self -> project-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.917 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2025-03-31 Mon 02:19:40:579.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> project-tag 2025-03-31 Mon 02:19:40:579.931 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2025-03-31 Mon 02:19:40:579.938 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.944 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-03-31 Mon 02:19:40:579.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.957 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.964 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:579.970 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:579.980 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:579.987 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2025-03-31 Mon 02:19:40:579.995 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.009 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.016 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2025-03-31 Mon 02:19:40:580.024 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-machine-interface 2025-03-31 Mon 02:19:40:580.031 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2025-03-31 Mon 02:19:40:580.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router : self -> logical-router-interface 2025-03-31 Mon 02:19:40:580.044 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-router : logical-router-interface -> nil 2025-03-31 Mon 02:19:40:580.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for physical-interface 2025-03-31 Mon 02:19:40:580.060 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2025-03-31 Mon 02:19:40:580.072 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:19:40:580.079 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:19:40:580.087 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for physical-interface 2025-03-31 Mon 02:19:40:580.095 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2025-03-31 Mon 02:19:40:580.104 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2025-03-31 Mon 02:19:40:580.111 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2025-03-31 Mon 02:19:40:580.118 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2025-03-31 Mon 02:19:40:580.125 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2025-03-31 Mon 02:19:40:580.134 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:19:40:580.141 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2025-03-31 Mon 02:19:40:580.147 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2025-03-31 Mon 02:19:40:580.154 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-03-31 Mon 02:19:40:580.161 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-03-31 Mon 02:19:40:580.167 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-03-31 Mon 02:19:40:580.175 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:19:40:580.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2025-03-31 Mon 02:19:40:580.190 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2025-03-31 Mon 02:19:40:580.196 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2025-03-31 Mon 02:19:40:580.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for logical-interface 2025-03-31 Mon 02:19:40:580.210 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2025-03-31 Mon 02:19:40:580.216 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2025-03-31 Mon 02:19:40:580.228 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2025-03-31 Mon 02:19:40:580.235 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for physical-router 2025-03-31 Mon 02:19:40:580.242 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2025-03-31 Mon 02:19:40:580.248 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2025-03-31 Mon 02:19:40:580.255 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2025-03-31 Mon 02:19:40:580.265 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for service-health-check 2025-03-31 Mon 02:19:40:580.272 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-03-31 Mon 02:19:40:580.279 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-03-31 Mon 02:19:40:580.286 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2025-03-31 Mon 02:19:40:580.294 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:580.301 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:19:40:580.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:19:40:580.318 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for service-health-check 2025-03-31 Mon 02:19:40:580.324 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-03-31 Mon 02:19:40:580.331 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-03-31 Mon 02:19:40:580.338 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2025-03-31 Mon 02:19:40:580.345 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.351 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.358 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.365 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2025-03-31 Mon 02:19:40:580.375 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-03-31 Mon 02:19:40:580.382 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-03-31 Mon 02:19:40:580.388 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-03-31 Mon 02:19:40:580.398 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for qos-config 2025-03-31 Mon 02:19:40:580.405 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2025-03-31 Mon 02:19:40:580.414 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2025-03-31 Mon 02:19:40:580.424 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for forwarding-class 2025-03-31 Mon 02:19:40:580.431 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2025-03-31 Mon 02:19:40:580.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2025-03-31 Mon 02:19:40:580.446 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2025-03-31 Mon 02:19:40:580.455 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:19:40:580.463 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2025-03-31 Mon 02:19:40:580.478 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2025-03-31 Mon 02:19:40:580.486 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2025-03-31 Mon 02:19:40:580.494 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:19:40:580.501 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2025-03-31 Mon 02:19:40:580.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2025-03-31 Mon 02:19:40:580.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2025-03-31 Mon 02:19:40:580.524 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:19:40:580.531 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2025-03-31 Mon 02:19:40:580.540 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2025-03-31 Mon 02:19:40:580.552 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2025-03-31 Mon 02:19:40:580.561 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2025-03-31 Mon 02:19:40:580.569 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for security-logging-object 2025-03-31 Mon 02:19:40:580.577 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2025-03-31 Mon 02:19:40:580.601 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2025-03-31 Mon 02:19:40:580.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2025-03-31 Mon 02:19:40:580.617 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2025-03-31 Mon 02:19:40:580.637 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for virtual-router 2025-03-31 Mon 02:19:40:580.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-router : virtual-router-sub-cluster -> self 2025-03-31 Mon 02:19:40:580.654 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap sub-cluster : self -> virtual-router-sub-cluster 2025-03-31 Mon 02:19:40:580.661 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap sub-cluster : bgp-router-sub-cluster -> virtual-router-sub-cluster 2025-03-31 Mon 02:19:40:580.669 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-router : self -> bgp-router-sub-cluster 2025-03-31 Mon 02:19:40:580.677 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bgp-router : bgp-router-sub-cluster -> nil 2025-03-31 Mon 02:19:40:580.689 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for bridge-domain 2025-03-31 Mon 02:19:40:580.697 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-03-31 Mon 02:19:40:580.705 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:19:40:580.713 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2025-03-31 Mon 02:19:40:580.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-03-31 Mon 02:19:40:580.728 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-03-31 Mon 02:19:40:580.736 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for bridge-domain 2025-03-31 Mon 02:19:40:580.743 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-03-31 Mon 02:19:40:580.754 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-03-31 Mon 02:19:40:580.762 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-03-31 Mon 02:19:40:580.771 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for application-policy-set 2025-03-31 Mon 02:19:40:580.777 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2025-03-31 Mon 02:19:40:580.785 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:580.792 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:580.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-03-31 Mon 02:19:40:580.806 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-03-31 Mon 02:19:40:580.816 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:19:40:580.823 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:19:40:580.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.838 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.853 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.862 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2025-03-31 Mon 02:19:40:580.869 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2025-03-31 Mon 02:19:40:580.878 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:19:40:580.885 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:19:40:580.892 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.899 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.918 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.926 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2025-03-31 Mon 02:19:40:580.933 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2025-03-31 Mon 02:19:40:580.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> address-group-tag 2025-03-31 Mon 02:19:40:580.948 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : address-group-tag -> nil 2025-03-31 Mon 02:19:40:580.957 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:19:40:580.964 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:19:40:580.972 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:580.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.001 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.008 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.014 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : self -> firewall-rule-tag 2025-03-31 Mon 02:19:40:581.021 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap tag : firewall-rule-tag -> nil 2025-03-31 Mon 02:19:40:581.027 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Updating dependency tacker rules for firewall-policy 2025-03-31 Mon 02:19:40:581.033 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-03-31 Mon 02:19:40:581.039 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.045 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.051 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-03-31 Mon 02:19:40:581.057 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2025-03-31 Mon 02:19:40:540.915 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2025-03-31 00:25:24.263439","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-8423-1.novalocal","build-id":"unknown","build-number":"unknown"}]} src/contrail-common/base/misc_utils.cc 41 2025-03-31 Mon 02:19:41:442.689 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.17:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:19:41:442.928 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Current receive sock buffer size is 262144 2025-03-31 Mon 02:19:41:443.030 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386648971008, Pid 1197922]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2025-03-31 Mon 02:19:41:462.170 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Vrouter family is 34 2025-03-31 Mon 02:19:41:462.303 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Flow table size : 20185088 2025-03-31 Mon 02:19:41:520.807 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Flow table size : 161218560 2025-03-31 Mon 02:19:41:520.847 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2025-03-31 Mon 02:19:41:520.862 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2025-03-31 Mon 02:19:41:520.891 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7fae4d400000 2025-03-31 Mon 02:19:41:520.898 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2025-03-31 Mon 02:19:41:520.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7fae43a00000 2025-03-31 Mon 02:19:41:520.918 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Sending Huge Page configuration to VROUTER 2025-03-31 Mon 02:19:41:520.968 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Vrouter using already allocated memory 2025-03-31 Mon 02:19:41:636.254 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Flow table size : 161218560 2025-03-31 Mon 02:19:41:636.403 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Flow table size : 20185088 2025-03-31 Mon 02:19:41:722.871 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: ROLE : Generator 2025-03-31 Mon 02:19:41:722.909 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: MODULE : contrail-vrouter-agent 2025-03-31 Mon 02:19:41:722.916 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-494-1 2025-03-31 Mon 02:19:41:722.922 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: NODE TYPE : Compute 2025-03-31 Mon 02:19:41:722.928 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: INSTANCE ID : 0 2025-03-31 Mon 02:19:41:722.934 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: HTTP SERVER PORT : 8085 2025-03-31 Mon 02:19:41:722.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: System Log Send Rate Limit: 0 -> 0 2025-03-31 Mon 02:19:41:723.383 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: HTTP Introspect Init 2025-03-31 Mon 02:19:41:723.639 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:19:41:723.720 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Sandesh Http Server Port: 8085 2025-03-31 Mon 02:19:41:723.791 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1192819.http_port 2025-03-31 Mon 02:19:41:723.975 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: CONNECT TO COLLECTOR: 1 2025-03-31 Mon 02:19:41:724.086 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:19:41:724.300 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: No Client: 1743387581724207 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:41:724.614 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvStart in state Idle 2025-03-31 Mon 02:19:41:724.712 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Disconnect 2025-03-31 Mon 02:19:41:726.094 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2025-03-31 Mon 02:19:41:726.126 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2025-03-31 Mon 02:19:41:726.174 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvCollectorUpdate in state Disconnect 2025-03-31 Mon 02:19:41:726.181 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2025-03-31 Mon 02:19:41:726.236 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2025-03-31 Mon 02:19:41:726.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2025-03-31 Mon 02:19:41:726.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface veth0ebd6937 flags 69699 2025-03-31 Mon 02:19:41:726.336 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface veth4dff224f flags 69699 2025-03-31 Mon 02:19:41:726.344 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface veth7454db11 flags 69699 2025-03-31 Mon 02:19:41:726.378 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vetha86214ed flags 69699 2025-03-31 Mon 02:19:41:726.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vetha1439fdb flags 69699 2025-03-31 Mon 02:19:41:726.396 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:19:41:726.419 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface veth3b613a14 flags 69699 2025-03-31 Mon 02:19:41:726.427 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vethc9574a7b flags 69699 2025-03-31 Mon 02:19:41:726.448 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:726.464 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vethb376b920 flags 69699 2025-03-31 Mon 02:19:41:726.471 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vethb28f070c flags 69699 2025-03-31 Mon 02:19:41:726.475 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581724695 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-03-31 Mon 02:19:41:726.533 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-03-31 Mon 02:19:41:726.557 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:726.607 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581724704 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:19:41:726.618 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:41:726.629 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:726.647 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581724736 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-03-31 Mon 02:19:41:726.662 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-03-31 Mon 02:19:41:726.671 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:726.844 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581726156 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:41:726.873 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:41:726.887 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:726.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581726225 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:41:726.993 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:41:727.005 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:727.029 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581726250 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:19:41:727.037 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:41:727.047 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:727.130 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387581726440 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:41:727.142 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:41:727.202 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface veth7646ab2c flags 69699 2025-03-31 Mon 02:19:41:727.211 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vethaeed3a20 flags 69699 2025-03-31 Mon 02:19:41:727.244 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vethc08d8cac flags 69699 2025-03-31 Mon 02:19:41:727.251 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2025-03-31 Mon 02:19:41:727.280 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2025-03-31 Mon 02:19:41:727.291 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2025-03-31 Mon 02:19:41:727.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2025-03-31 Mon 02:19:41:727.310 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2025-03-31 Mon 02:19:41:727.317 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2025-03-31 Mon 02:19:41:727.374 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface lo 2025-03-31 Mon 02:19:41:727.387 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-03-31 Mon 02:19:41:727.398 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2025-03-31 Mon 02:19:41:727.410 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface docker0 2025-03-31 Mon 02:19:41:727.421 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-03-31 Mon 02:19:41:727.441 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface lo 2025-03-31 Mon 02:19:41:727.452 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-03-31 Mon 02:19:41:727.463 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface ftun0 2025-03-31 Mon 02:19:41:727.474 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-03-31 Mon 02:19:41:727.485 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink address message RTM_NEWADDR for interface pkt0 2025-03-31 Mon 02:19:41:727.543 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.560 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.573 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.586 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.614 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.627 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.640 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 10.94.185.0/24 Interface lxdbr0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.646 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.663 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.676 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2025-03-31 Mon 02:19:41:727.682 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.688 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:19:41:727.694 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.700 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:19:41:727.706 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.711 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.717 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:19:41:727.723 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.729 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:19:41:727.735 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-03-31 Mon 02:19:41:727.741 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-03-31 Mon 02:19:41:727.758 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.765 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.771 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.782 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-03-31 Mon 02:19:41:727.788 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:19:41:727.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:19:41:727.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:19:41:727.805 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-03-31 Mon 02:19:41:727.814 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:19:41:727.820 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:19:41:727.826 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-03-31 Mon 02:19:41:727.831 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:19:41:727.837 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:19:41:727.843 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:19:41:727.849 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:19:41:727.855 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-03-31 Mon 02:19:41:771.273 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:19:41:771.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Router ID configured. Connection to controller initiated 2025-03-31 Mon 02:19:41:777.341 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:778.813 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581777262 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:778.830 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:778.847 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:778.876 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581777330 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2025-03-31 Mon 02:19:41:778.883 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:41:778.892 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:778.901 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581777538 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:778.913 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:778.920 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:778.936 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581777555 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, XMPP:dns-server:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:19:41:778.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:41:778.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server [fe80::f816:3eff:fec0:9c69%vhost0]:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-03-31 Mon 02:19:41:829.938 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:829.979 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581829738 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:829.987 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:829.994 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:830.006 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581829896 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 35487 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:830.011 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:830.164 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:830.186 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581830021 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:830.193 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:830.201 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:830.217 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387581830151 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 33273 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:830.223 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:832.563 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:35487::10.0.0.17:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-03-31 Mon 02:19:41:832.602 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-03-31 Mon 02:19:41:840.605 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:840.646 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387581840559 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 35487 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:840.655 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:843.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:33273::10.0.0.17:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-03-31 Mon 02:19:41:850.274 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-03-31 Mon 02:19:41:850.482 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:850.513 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387581850464 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 33273 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:850.522 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:855.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:19:41:855.642 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:855.658 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387581855550 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:855.672 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:855.680 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:887.231 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387581855576 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:887.273 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:894.744 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:19:41:894.786 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:894.803 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387581894701 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:894.811 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:905.264 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:19:41:905.307 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:905.324 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387581905213 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:905.332 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:905.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:905.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387581905239 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:905.354 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:41:988.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-03-31 Mon 02:19:41:989.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:41:989.109 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387581989069 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:41:989.117 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:42:010.366 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-03-31 Mon 02:19:42:010.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:010.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582010336 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:42:010.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:42:028.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:028.705 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582028628 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:42:028.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:42:028.773 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:028.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582028745 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:19:42:028.801 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:42:031.462 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-494-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-03-31 Mon 02:19:42:034.115 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:034.150 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387582034092 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:42:034.169 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:42:067.481 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:067.523 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582067349 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-494-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-03-31 Mon 02:19:42:067.532 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-03-31 Mon 02:19:42:067.539 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:067.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582067395 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-03-31 Mon 02:19:42:067.566 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:42:149.258 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:149.575 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387582149180 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:42:149.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:42:187.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:187.360 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387582187237 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:42:187.369 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:42:190.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:190.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387582187516 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:19:42:190.483 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:19:42:209.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:209.315 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387582208924 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric acl_uuid = 57fa88d5-1162-43c2-ba82-a5761f77fd36 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:19:42:209.325 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:19:42:240.544 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:42:240.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387582240362 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f name = default-domain:default-project:ip-fabric acl_uuid = 57fa88d5-1162-43c2-ba82-a5761f77fd36 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-03-31 Mon 02:19:42:240.621 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-03-31 Mon 02:19:42:726.502 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:42:726.575 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:42:726.585 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:19:42:727.315 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Idle 2025-03-31 Mon 02:19:42:727.417 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:42:727.445 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387582726626 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:42:727.453 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:42:727.462 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:19:42:727.500 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:42:727.647 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387582727407 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 34, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:42:727.665 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:43:735.220 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:19:43:735.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:19:43:735.572 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:43:735.614 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387583735312 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:43:735.622 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:43:735.630 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:43:735.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387583735559 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 36, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:43:735.787 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:45:735.593 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:45:735.716 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:45:735.870 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:19:45:736.355 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Idle 2025-03-31 Mon 02:19:45:738.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:45:738.162 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387585735906 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:45:738.176 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:45:738.185 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:19:45:738.223 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:45:738.363 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387585738100 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 38, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:45:738.383 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:47:736.585 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:19:47:736.865 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:19:47:736.967 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:47:738.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387587736691 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:47:738.193 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:47:738.203 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:47:738.348 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387587736955 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 40, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:47:738.369 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:47:753.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38994 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:19:47:809.111 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38994 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:19:51:736.961 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:51:737.109 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:19:51:737.144 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:19:51:737.243 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Idle 2025-03-31 Mon 02:19:51:737.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:51:737.366 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387591737185 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:51:737.384 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:51:737.391 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:19:51:737.430 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:19:51:737.578 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387591737325 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 42, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:51:737.596 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:19:55:737.370 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:19:55:737.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:19:55:737.733 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:55:737.760 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387595737450 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:19:55:737.768 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:19:55:737.778 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:19:55:737.936 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387595737721 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 44, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:19:55:737.957 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:03:737.759 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:20:03:737.843 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:20:03:737.884 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:20:03:737.987 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:20:03:738.115 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:03:738.155 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387603737925 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:20:03:738.168 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:20:03:738.179 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:20:03:738.238 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:03:738.489 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387603738099 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 46, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:03:738.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:10:541.378 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:541.461 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387610541339 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric total_acl_rules = 4 interface_list= [ [ (*_iter66) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0, ] ] virtualmachine_list= [ [ ] ] acl = default-domain:default-project:ip-fabric:ip-fabric mirror_acl = ] 2025-03-31 Mon 02:20:10:541.475 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:20:10:541.496 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:541.523 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387610541390 [SYS_NOTICE]: UveVMInterfaceAgentTrace: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 ip_address = 10.0.0.17 virtual_network = default-domain:default-project:ip-fabric mac_address = fa:16:3e:c0:9c:69 label = 17 active = 1 vm_name = l2_active = 1 uuid = cca6d80d-e05d-4411-8298-d2b2a4db205d ip6_address = :: ip6_active = 1 floating_ips= [ [ ] ] vm_uuid = ip4_active = 1 is_health_check_active = 1 health_check_instance_list= [ [ ] ] admin_state = 1 fixed_ip4_list= [ [ (*_iter26) = 10.0.0.17, ] ] fixed_ip6_list= [ [ ] ] vn_uuid = 3da78d9c-3d98-4934-a990-647a4cdbfe7f tx_vlan = 65535 rx_vlan = 65535 parent_interface = ens3 vhostuser_mode = 0 port_mirror_enabled = 0 ] 2025-03-31 Mon 02:20:10:541.530 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVMInterfaceAgentTrace 2025-03-31 Mon 02:20:10:541.539 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:541.550 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387610541397 [SYS_NOTICE]: UVEVMITags: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 app = tier = site = deployment = labels= [ [ ] ] custom_tags= [ [ ] ] ] 2025-03-31 Mon 02:20:10:541.556 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMITags 2025-03-31 Mon 02:20:10:542.877 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:542.953 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610542845 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric associated_fip_count = 0 in_bandwidth_usage = 0 out_bandwidth_usage = 0 vrf_stats_list= [ [ [ name = default-domain:default-project:ip-fabric:ip-fabric diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:20:10:542.963 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:20:10:543.018 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:543.061 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610542863 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2025-03-31 Mon 02:20:10:543.068 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:20:10:543.075 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:543.209 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610542989 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric port_bucket_bmap= [ udp_sport_bitmap= [ [ (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, ] ] udp_dport_bitmap= [ [ (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, ] ] tcp_sport_bitmap= [ [ (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, ] ] tcp_dport_bitmap= [ [ (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, ] ] ] raw_if_stats= [ in_pkts = 161 in_bytes = 13275 out_pkts = 160 out_bytes = 12961 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 0 out_bw_usage = 0 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:20:10:543.224 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:20:10:543.248 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:543.259 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610543010 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:20:10:543.265 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:20:10:543.271 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:543.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610543054 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 connected_networks= [ [ (*_iter142) = default-domain:default-project:ip-fabric, ] ] vn_count = 1 ] 2025-03-31 Mon 02:20:10:543.288 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:20:10:543.296 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:10:543.309 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387610543118 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 interface_list= [ [ (*_iter143) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0, ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ ] ] vmi_count= [ active = 1 ] down_interface_count = 0 ] 2025-03-31 Mon 02:20:10:543.314 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:20:11:724.585 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:11:724.685 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387611724437 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 3 dequeues = 3 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 57 dequeues = 57 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 70 dequeues = 70 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387603737985 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387610543295 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:20:11:724.699 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:11:724.709 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:11:724.962 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387611724539 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 57, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:11:725.041 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:11:729.508 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:11:729.620 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387611729440 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 xmpp_peer_list= [ [ [ ip = 10.0.0.17 status = 1 setup_time = 1743387582067399 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.17, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2025-03-31 00:25:24.263439","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-8423-1.novalocal","build-id":"unknown","build-number":"unknown"}]} phy_if= [ [ [ name = ens3 mac_address = fa:16:3e:c0:9c:69 ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:c0:9c:69 ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_DEBUG log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.17, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.17 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = ens3, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.17 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.17:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.17:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-juju-os-494-1 flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.17:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2025-03-31 00:22:50.809725", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-8423-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2025-03-31 Mon 02:20:11:729.651 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-03-31 Mon 02:20:11:729.666 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:11:729.927 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387611729513 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 161 in_bytes = 13275 out_tpkts = 160 out_bytes = 12961 exception_packets = 20 exception_packets_dropped = 10 exception_packets_allowed = 10 total_flows = 0 aged_flows = 0 uptime = 1743387580541150 phy_if_info= [ [ _iter254->first = ens3 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 161 in_bytes = 13275 out_pkts = 160 out_bytes = 12961 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 135 in_bytes = 11911 out_pkts = 162 out_bytes = 13269 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:20:11:729.954 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:20:11:729.966 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:11:730.010 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387611729579 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:20:11:730.020 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:20:11:738.117 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:20:11:738.472 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:20:11:738.618 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:20:11:738.659 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387611738268 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:20:11:738.679 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:20:11:738.690 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:20:11:739.199 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387611738593 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 62, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:11:739.248 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:24:666.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:56346 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:20:27:738.625 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:20:27:738.712 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:20:27:738.751 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:20:27:738.833 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Idle 2025-03-31 Mon 02:20:27:738.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:27:738.968 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387627738786 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:20:27:738.976 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:20:27:738.983 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:20:27:739.013 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:27:739.226 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387627738925 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 64, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:27:739.255 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:40:544.780 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:40:544.872 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387640544740 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:20:40:544.888 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:20:40:544.972 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:40:545.135 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387640544948 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 207 in_bytes = 16648 out_pkts = 206 out_bytes = 16226 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 899 out_bw_usage = 870 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:20:40:545.149 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:20:40:545.158 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:40:545.168 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387640544954 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:20:40:545.175 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:20:41:724.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:725.083 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641724687 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 69 dequeues = 69 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 85 dequeues = 85 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387627738832 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387640545157 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:20:41:725.110 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:41:725.118 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:725.367 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641724882 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 69, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:41:725.391 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:20:41:730.510 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:730.601 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641730443 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.6125 five_min_avg = 0.565 fifteen_min_avg = 0.4075 ] cpu_share = 0.474789 ] ] 2025-03-31 Mon 02:20:41:730.613 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:20:41:730.626 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:730.646 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641730496 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.474789 used_sys_mem = 4269468306 one_min_cpuload = 0.6125 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:20:41:730.655 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:20:41:730.664 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:730.835 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641730525 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 207 in_bytes = 16648 out_tpkts = 206 out_bytes = 16226 exception_packets = 21 exception_packets_dropped = 11 raw_vhost_stats= [ name = vhost0 in_pkts = 207 in_bytes = 16648 out_pkts = 206 out_bytes = 16226 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 166 in_bytes = 14546 out_pkts = 208 out_bytes = 16642 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:20:41:730.859 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:20:41:730.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:20:41:730.893 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387641730550 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:20:41:730.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:20:43:738.995 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:20:43:739.257 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:20:43:739.380 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:20:43:739.416 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387643739081 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:20:43:739.425 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:20:43:739.434 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:20:43:739.648 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387643739354 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 75, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:20:43:739.672 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:10:544.879 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:10:545.075 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387670544671 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:21:10:545.115 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:21:10:545.128 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:10:545.197 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387670544934 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 225 in_bytes = 17584 out_pkts = 224 out_bytes = 17054 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 258 out_bw_usage = 228 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:21:10:545.210 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:21:10:545.218 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:10:545.230 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387670544940 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:21:10:545.237 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:21:11:725.241 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:11:725.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387671725089 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 80 dequeues = 80 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 97 dequeues = 97 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387643739088 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387670545217 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:21:11:725.424 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:11:725.434 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:11:725.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387671725193 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 80, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:21:11:725.683 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:11:730.802 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:11:730.995 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387671730766 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 225 in_bytes = 17584 out_tpkts = 224 out_bytes = 17054 exception_packets = 22 exception_packets_dropped = 12 raw_vhost_stats= [ name = vhost0 in_pkts = 225 in_bytes = 17584 out_pkts = 224 out_bytes = 17054 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 2.04372e-15 total_out_bandwidth_utilization = 3.11166e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 377, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 574, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 169 in_bytes = 14744 out_pkts = 226 out_bytes = 17578 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:21:11:731.031 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:21:11:731.045 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:11:731.072 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387671730812 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:21:11:731.079 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:21:13:739.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:21:13:739.464 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:21:13:739.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:21:13:739.591 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Idle 2025-03-31 Mon 02:21:13:739.686 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:13:739.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387673739501 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:21:13:739.722 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:21:13:739.758 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:21:13:739.786 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:13:740.000 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387673739669 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 84, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:21:13:740.021 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:25:562.334 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:42098 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:21:40:544.664 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:40:544.722 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387700544630 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:21:40:544.747 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:21:40:544.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:40:545.117 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387700544918 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 244 in_bytes = 18562 out_pkts = 243 out_bytes = 17924 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 269 out_bw_usage = 240 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:21:40:545.137 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:21:40:545.146 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:40:545.157 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387700544927 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:21:40:545.164 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:21:41:725.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:725.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701725306 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 89 dequeues = 89 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 108 dequeues = 108 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387673739589 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387700545145 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:21:41:725.588 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:41:725.612 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:725.885 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701725419 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 89, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:21:41:725.904 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:21:41:731.088 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:731.151 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701731060 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.4975 five_min_avg = 0.5425 fifteen_min_avg = 0.41 ] cpu_share = 0.40625 ] ] 2025-03-31 Mon 02:21:41:731.161 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:21:41:731.170 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:731.184 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701731076 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.40625 used_sys_mem = 4269468306 one_min_cpuload = 0.4975 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:21:41:731.191 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:21:41:731.198 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:731.358 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701731103 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 244 in_bytes = 18562 out_tpkts = 243 out_bytes = 17924 raw_vhost_stats= [ name = vhost0 in_pkts = 244 in_bytes = 18562 out_pkts = 243 out_bytes = 17924 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 173 in_bytes = 14984 out_pkts = 245 out_bytes = 18556 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:21:41:731.380 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:21:41:731.391 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:21:41:731.429 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387701731137 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:21:41:731.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:21:43:739.752 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:21:43:740.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:21:43:740.146 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:43:740.173 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387703739863 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:21:43:740.180 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:21:43:740.188 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:21:43:740.422 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387703740131 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 95, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:21:43:740.447 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:10:544.805 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:10:544.864 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387730544773 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:22:10:544.875 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:22:10:544.915 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:10:544.977 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387730544907 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 263 in_bytes = 19540 out_pkts = 272 out_bytes = 19650 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] out_bw_usage = 476 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:22:10:545.725 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:22:10:545.754 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:10:545.768 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387730544914 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:22:10:545.775 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:22:11:725.795 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:11:725.908 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387731725607 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 100 dequeues = 100 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 120 dequeues = 120 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387703739871 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387730545752 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:22:11:725.933 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:11:725.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:11:726.180 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387731725721 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 100, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:22:11:726.197 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:11:731.338 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:11:731.637 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387731731299 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 263 in_bytes = 19540 out_tpkts = 272 out_bytes = 19650 exception_packets = 23 exception_packets_dropped = 13 raw_vhost_stats= [ name = vhost0 in_pkts = 263 in_bytes = 19540 out_pkts = 272 out_bytes = 19650 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 9.6494e-16 total_out_bandwidth_utilization = 1.40946e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 178, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 260, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 187 in_bytes = 16080 out_pkts = 264 out_bytes = 19534 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:22:11:731.671 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:22:11:731.685 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:11:731.713 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387731731349 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:22:11:731.720 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:22:13:740.174 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:22:13:740.257 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:22:13:740.267 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:22:13:740.351 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Idle 2025-03-31 Mon 02:22:13:740.477 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:13:740.517 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387733740299 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:22:13:740.525 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:22:13:740.533 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:22:13:740.567 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:13:740.835 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387733740459 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 17, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 104, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:22:13:740.859 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:26:542.054 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:36924 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:22:40:546.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:40:546.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387760546437 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:22:40:546.545 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:22:40:546.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:40:546.616 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387760546549 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 282 in_bytes = 20518 out_pkts = 299 out_bytes = 20912 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 260 out_bw_usage = 336 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:22:40:546.639 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:22:40:546.647 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:40:546.657 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387760546555 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:22:40:546.663 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:22:41:726.009 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:726.097 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387761725842 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 109 dequeues = 109 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 131 dequeues = 131 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387733740350 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387760546645 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:22:41:726.113 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:41:726.124 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:726.357 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387761725959 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 109, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:22:41:726.383 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:22:41:731.527 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:731.600 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387761731489 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.32125 five_min_avg = 0.48125 fifteen_min_avg = 0.3975 ] cpu_share = 0.408333 ] ] 2025-03-31 Mon 02:22:41:731.610 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:22:41:731.619 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:731.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387761731506 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.408333 used_sys_mem = 4269468306 one_min_cpuload = 0.32125 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:22:41:731.642 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:22:41:731.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:731.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387761731546 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 282 in_bytes = 20518 out_tpkts = 299 out_bytes = 20912 raw_vhost_stats= [ name = vhost0 in_pkts = 282 in_bytes = 20518 out_pkts = 299 out_bytes = 20912 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 199 in_bytes = 16712 out_pkts = 283 out_bytes = 20512 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:22:41:731.856 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:22:41:731.864 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:22:41:731.892 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387761731576 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:22:41:731.898 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:22:43:740.497 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:22:43:741.512 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:22:43:741.696 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:43:741.753 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387763740599 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:22:43:741.764 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:22:43:741.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:22:43:742.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387763741672 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 115, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:22:43:742.163 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:10:546.663 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:10:546.778 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387790546618 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:23:10:546.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:23:10:547.203 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:10:547.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387790547175 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 300 in_bytes = 21454 out_pkts = 320 out_bytes = 21950 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 249 out_bw_usage = 276 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:23:10:547.381 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:23:10:547.395 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:10:547.410 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387790547184 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:23:10:547.422 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:23:11:726.287 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:11:726.413 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387791726131 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 120 dequeues = 120 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 143 dequeues = 143 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387763740608 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387790547393 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:23:11:726.428 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:11:726.437 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:11:726.796 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743387791726239 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 120, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:23:11:726.825 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:11:731.834 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:11:732.020 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387791731800 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 300 in_bytes = 21454 out_tpkts = 320 out_bytes = 21950 raw_vhost_stats= [ name = vhost0 in_pkts = 300 in_bytes = 21454 out_pkts = 320 out_bytes = 21950 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.48099e-16 total_out_bandwidth_utilization = 1.38236e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 138, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 255, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 205 in_bytes = 17120 out_pkts = 301 out_bytes = 21448 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:23:11:732.046 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:23:11:732.056 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:11:732.082 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387791731852 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:23:11:732.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:23:13:741.662 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:23:13:741.821 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:23:13:741.867 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:23:13:741.956 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Idle 2025-03-31 Mon 02:23:13:742.058 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:13:742.084 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387793741905 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:23:13:742.101 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:23:13:742.108 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:23:13:742.141 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:13:742.370 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387793742047 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 124, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:23:13:742.392 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:27:443.720 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:32920 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:23:40:516.708 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:50254 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:23:40:548.224 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:40:548.280 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387820548190 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:23:40:548.290 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:23:40:548.459 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:40:548.599 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387820548437 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 324 in_bytes = 22835 out_pkts = 356 out_bytes = 23854 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 368 out_bw_usage = 507 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:23:40:548.633 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:23:40:548.662 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:40:548.678 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387820548445 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:23:40:548.689 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:23:40:566.069 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:50254 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:23:41:726.518 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:726.608 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387821726346 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 129 dequeues = 129 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 154 dequeues = 154 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387793741955 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387820548660 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:23:41:726.622 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:41:726.631 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:726.862 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387821726471 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 21, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 129, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:23:41:726.893 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:41:732.067 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:732.140 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387821732037 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.24625 five_min_avg = 0.42875 fifteen_min_avg = 0.38375 ] cpu_share = 0.408333 ] ] 2025-03-31 Mon 02:23:41:732.151 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:23:41:732.161 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:732.177 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387821732052 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.408333 used_sys_mem = 4269468306 one_min_cpuload = 0.24625 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:23:41:732.183 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:23:41:732.191 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:732.333 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387821732081 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 324 in_bytes = 22835 out_tpkts = 356 out_bytes = 23854 raw_vhost_stats= [ name = vhost0 in_pkts = 324 in_bytes = 22835 out_pkts = 356 out_bytes = 23854 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 226 in_bytes = 18394 out_pkts = 325 out_bytes = 22829 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:23:41:732.356 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:23:41:732.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:23:41:732.390 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387821732118 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:23:41:732.397 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:23:43:742.080 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:23:43:742.294 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:23:43:742.404 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:43:742.431 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387823742172 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:23:43:742.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:23:43:742.446 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:23:43:742.667 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387823742392 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 21, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 135, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:23:43:742.688 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:23:49:120.254 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:32788 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:23:49:161.946 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:32788 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:24:04:625.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53642 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:24:04:673.824 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53642 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:24:10:548.410 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:10:548.471 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387850548375 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:24:10:548.502 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:24:10:548.593 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:10:548.681 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387850548570 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 436 in_bytes = 32158 out_pkts = 483 out_bytes = 34135 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2486 out_bw_usage = 2741 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:24:10:548.695 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:24:10:548.705 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:10:548.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387850548578 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:24:10:548.751 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:24:11:726.771 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:11:726.897 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387851726643 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 140 dequeues = 140 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 166 dequeues = 166 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387823742183 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387850548704 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:24:11:726.924 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:24:11:726.933 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:11:727.152 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387851726719 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 140, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:24:11:727.169 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:24:11:732.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:11:732.542 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387851732275 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 436 in_bytes = 32158 out_tpkts = 483 out_bytes = 34135 exception_packets = 24 exception_packets_allowed = 11 raw_vhost_stats= [ name = vhost0 in_pkts = 436 in_bytes = 32158 out_pkts = 483 out_bytes = 34135 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.89299e-15 total_out_bandwidth_utilization = 7.73578e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1456, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1427, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 338 in_bytes = 28045 out_pkts = 437 out_bytes = 32152 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:24:11:732.570 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:24:11:732.580 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:11:732.608 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387851732315 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:24:11:732.615 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:24:13:742.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:24:13:742.512 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:24:13:742.544 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:24:13:742.650 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Idle 2025-03-31 Mon 02:24:13:742.753 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:13:742.780 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387853742597 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:24:13:742.788 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:24:13:742.795 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:24:13:742.824 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:13:743.034 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387853742743 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 144, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:24:13:743.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:24:14:204.452 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:42936 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:24:14:249.890 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:42936 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:24:28:411.262 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41906 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:24:40:548.489 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:40:548.554 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387880548454 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:24:40:548.565 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:24:40:548.781 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:40:548.881 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387880548756 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 449 in_bytes = 33049 out_pkts = 501 out_bytes = 35156 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 237 out_bw_usage = 272 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:24:40:548.895 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:24:40:548.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:40:548.913 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743387880548766 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:24:40:548.920 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:24:41:726.993 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:727.073 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387881726839 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 149 dequeues = 149 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 177 dequeues = 177 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387853742648 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387880548902 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:24:41:727.094 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:24:41:727.102 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:727.357 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387881726940 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 149, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:24:41:727.377 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:24:41:732.532 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:732.600 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387881732498 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.655 five_min_avg = 0.545 fifteen_min_avg = 0.42875 ] cpu_share = 0.2875 ] ] 2025-03-31 Mon 02:24:41:732.610 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:24:41:732.619 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:732.633 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387881732513 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.2875 used_sys_mem = 4269468306 one_min_cpuload = 0.655 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:24:41:732.640 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:24:41:732.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:732.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387881732542 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 449 in_bytes = 33049 out_tpkts = 501 out_bytes = 35156 exception_packets = 25 exception_packets_dropped = 14 raw_vhost_stats= [ name = vhost0 in_pkts = 449 in_bytes = 33049 out_pkts = 501 out_bytes = 35156 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 350 in_bytes = 28814 out_pkts = 450 out_bytes = 33043 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:24:41:732.931 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:24:41:732.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:24:41:732.968 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387881732568 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:24:41:732.996 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:24:43:742.793 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:24:43:743.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:24:43:743.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:43:743.184 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387883742873 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:24:43:743.192 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:24:43:743.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:24:43:743.422 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387883743131 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 155, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:24:43:743.447 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:10:548.562 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:10:548.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387910548524 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:25:10:548.661 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:25:10:548.674 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:10:548.792 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387910548659 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 456 in_bytes = 33523 out_pkts = 511 out_bytes = 35676 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 138 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:25:10:548.808 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:25:10:548.821 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:10:548.836 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387910548663 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:25:10:548.845 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:25:11:727.300 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:11:727.433 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387911727148 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 160 dequeues = 160 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 189 dequeues = 189 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387883742880 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387910548819 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:25:11:727.464 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:11:727.476 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:11:727.711 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387911727250 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 160, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:25:11:727.751 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:11:732.795 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:11:733.011 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387911732758 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 456 in_bytes = 33523 out_tpkts = 511 out_bytes = 35676 exception_packets = 27 exception_packets_allowed = 13 raw_vhost_stats= [ name = vhost0 in_pkts = 456 in_bytes = 33523 out_pkts = 511 out_bytes = 35676 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 461 out_bandwidth_usage = 542 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 8.40257e-16 total_out_bandwidth_utilization = 1.04626e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 155, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 193, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 357 in_bytes = 29208 out_pkts = 459 out_bytes = 33601 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:25:11:733.041 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:25:11:733.051 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:11:733.078 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387911732810 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:25:11:733.086 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:25:13:743.192 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:25:13:743.260 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:25:13:743.269 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:25:13:743.365 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Idle 2025-03-31 Mon 02:25:13:743.461 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:13:743.487 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387913743296 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:25:13:743.495 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:25:13:743.503 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:25:13:743.575 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:13:743.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387913743449 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 28, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 164, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:25:13:743.898 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:29:298.018 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53298 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:25:40:550.126 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:40:550.184 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387940550095 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:25:40:550.195 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:25:40:551.073 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:40:551.167 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387940550206 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 463 in_bytes = 33997 out_pkts = 520 out_bytes = 36182 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] out_bw_usage = 134 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:25:40:551.180 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:25:40:551.187 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:40:551.198 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387940550209 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:25:40:551.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:25:41:727.505 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:727.586 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387941727369 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 169 dequeues = 169 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 200 dequeues = 200 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387913743364 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743387940551186 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:25:41:727.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:41:727.632 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:727.860 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387941727480 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 169, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:25:41:727.877 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:25:41:733.051 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:733.143 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387941733017 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.40875 five_min_avg = 0.49 fifteen_min_avg = 0.41625 ] cpu_share = 0.235417 ] ] 2025-03-31 Mon 02:25:41:733.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:25:41:733.168 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:733.185 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387941733031 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.235417 used_sys_mem = 4269468306 one_min_cpuload = 0.40875 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:25:41:733.192 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:25:41:733.199 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:733.362 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387941733059 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 463 in_bytes = 33997 out_tpkts = 520 out_bytes = 36182 raw_vhost_stats= [ name = vhost0 in_pkts = 463 in_bytes = 33997 out_pkts = 520 out_bytes = 36182 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 363 in_bytes = 29588 out_pkts = 466 out_bytes = 34075 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:25:41:733.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:25:41:733.396 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:25:41:733.424 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743387941733094 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:25:41:733.433 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:25:43:743.507 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:25:43:743.756 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:25:43:743.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:43:743.888 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387943743582 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:25:43:743.900 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:25:43:743.910 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:25:43:744.123 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743387943743841 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 175, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:25:43:744.148 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:10:551.627 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:10:551.700 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387970551587 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:26:10:551.715 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:26:10:551.724 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:10:551.824 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387970551713 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 498 in_bytes = 36908 out_pkts = 555 out_bytes = 39053 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 776 out_bw_usage = 765 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:26:10:551.837 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:26:10:551.847 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:10:551.857 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387970551720 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:26:10:551.863 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:26:11:727.780 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:11:727.890 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387971727630 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 180 dequeues = 180 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 212 dequeues = 212 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743387943743589 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743387970551845 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:26:11:727.917 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:11:727.934 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:11:728.145 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743387971727717 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 180, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:26:11:728.161 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:11:733.283 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:11:733.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387971733252 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 498 in_bytes = 36908 out_tpkts = 555 out_bytes = 39053 exception_packets = 28 exception_packets_allowed = 14 raw_vhost_stats= [ name = vhost0 in_pkts = 498 in_bytes = 36908 out_pkts = 555 out_bytes = 39053 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 2.25514e-15 total_out_bandwidth_utilization = 2.44488e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 416, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 451, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 395 in_bytes = 32333 out_pkts = 501 out_bytes = 36986 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:26:11:733.492 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:26:11:733.501 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:11:733.530 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743387971733298 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:26:11:733.536 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:26:13:743.900 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:26:13:743.996 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:26:13:744.035 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:26:13:744.147 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Idle 2025-03-31 Mon 02:26:13:744.244 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:13:744.271 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387973744086 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:26:13:744.279 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:26:13:744.287 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:26:13:744.321 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:13:744.580 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743387973744232 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 18 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 29, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 184, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:26:13:744.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:30:190.005 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:52434 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:26:40:560.446 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:40:560.756 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388000560401 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:26:40:560.771 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:26:40:561.435 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:40:561.563 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388000561398 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 503 in_bytes = 37238 out_pkts = 560 out_bytes = 39311 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 88 out_bw_usage = 68 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:26:40:563.321 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:26:40:563.350 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:40:563.370 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388000561413 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:26:40:563.379 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:26:41:728.002 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:728.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388001727878 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 189 dequeues = 189 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 223 dequeues = 223 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743387973744145 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388000563348 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:26:41:728.106 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:41:728.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:728.409 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388001727976 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 19 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 189, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:26:41:728.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:26:41:733.495 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:733.567 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388001733459 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.35875 five_min_avg = 0.46625 fifteen_min_avg = 0.41375 ] cpu_share = 0.2375 ] ] 2025-03-31 Mon 02:26:41:733.578 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:26:41:733.587 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:733.602 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388001733481 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.2375 used_sys_mem = 4269468306 one_min_cpuload = 0.35875 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:26:41:733.609 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:26:41:733.616 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:733.787 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388001733540 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 503 in_bytes = 37238 out_tpkts = 560 out_bytes = 39311 raw_vhost_stats= [ name = vhost0 in_pkts = 503 in_bytes = 37238 out_pkts = 560 out_bytes = 39311 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 397 in_bytes = 32465 out_pkts = 506 out_bytes = 37316 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:26:41:733.811 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:26:41:733.821 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:26:41:733.848 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388001733588 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:26:41:733.855 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:26:43:744.299 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:26:43:744.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:26:43:744.643 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:43:744.672 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388003744376 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:26:43:744.679 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:26:43:744.687 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:26:43:744.908 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388003744631 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 19 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 195, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:26:43:744.934 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:10:553.442 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:10:553.528 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388030553406 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:27:10:553.541 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:27:10:553.550 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:10:553.611 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388030553515 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 510 in_bytes = 37712 out_pkts = 570 out_bytes = 39887 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 130 out_bw_usage = 158 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:27:10:553.635 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:27:10:553.642 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:10:553.653 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388030553519 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:27:10:553.659 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:27:11:728.274 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:11:728.394 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388031728159 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 200 dequeues = 200 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 235 dequeues = 235 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388003744385 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388030553641 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:27:11:728.408 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:11:728.435 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:11:728.668 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388031728232 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 19 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 200, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:27:11:728.696 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:11:733.788 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:11:733.993 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388031733748 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 510 in_bytes = 37712 out_tpkts = 570 out_bytes = 39887 raw_vhost_stats= [ name = vhost0 in_pkts = 510 in_bytes = 37712 out_pkts = 570 out_bytes = 39887 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 4.17418e-16 total_out_bandwidth_utilization = 5.80048e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 77, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 107, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 404 in_bytes = 32915 out_pkts = 513 out_bytes = 37790 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:27:11:734.022 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:27:11:734.035 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:11:734.096 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388031733797 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:27:11:734.105 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:27:13:744.706 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:27:13:744.808 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:27:13:744.845 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:27:13:744.953 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Idle 2025-03-31 Mon 02:27:13:745.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:13:745.127 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388033744886 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:27:13:745.158 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:27:13:745.170 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:27:13:745.202 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:13:745.557 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388033745067 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 19 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 30, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 204, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:27:13:745.588 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:31:078.045 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:42964 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:27:40:555.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:40:555.517 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388060555382 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:27:40:555.527 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:27:40:555.704 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:40:555.882 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388060555678 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 517 in_bytes = 38186 out_pkts = 577 out_bytes = 40253 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:27:40:555.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:27:40:555.996 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:40:556.013 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388060555687 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:27:40:556.023 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:27:41:728.521 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:728.616 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388061728344 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 209 dequeues = 209 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 246 dequeues = 246 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388033744951 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388060555994 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:27:41:728.629 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:41:728.638 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:728.887 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388061728466 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 209, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:27:41:728.922 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:27:41:733.992 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:734.093 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388061733957 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.3 five_min_avg = 0.42625 fifteen_min_avg = 0.4025 ] cpu_share = 0.233333 ] ] 2025-03-31 Mon 02:27:41:734.105 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:27:41:734.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:734.130 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388061733978 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.233333 used_sys_mem = 4269468306 one_min_cpuload = 0.3 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:27:41:734.137 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:27:41:734.144 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:734.302 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388061734033 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 517 in_bytes = 38186 out_tpkts = 577 out_bytes = 40253 raw_vhost_stats= [ name = vhost0 in_pkts = 517 in_bytes = 38186 out_pkts = 577 out_bytes = 40253 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 408 in_bytes = 33155 out_pkts = 520 out_bytes = 38264 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:27:41:734.327 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:27:41:734.338 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:27:41:734.364 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388061734085 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:27:41:734.371 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:27:43:745.114 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:27:43:745.332 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:27:43:745.423 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:43:745.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388063745207 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:27:43:745.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:27:43:745.482 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:27:43:745.694 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388063745412 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 215, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:27:43:745.713 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:02:912.400 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:60032 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:28:02:957.881 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:60032 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:28:10:555.321 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:10:555.376 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388090555281 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:28:10:555.386 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:28:10:555.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:10:555.631 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388090555406 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 530 in_bytes = 39077 out_pkts = 590 out_bytes = 41036 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 245 out_bw_usage = 216 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:28:10:555.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:28:10:555.669 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:10:555.680 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388090555471 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:28:10:555.688 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:28:11:416.550 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38968 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:28:11:461.937 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38968 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:28:11:728.818 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:11:728.961 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388091728642 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 220 dequeues = 220 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 258 dequeues = 258 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388063745214 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388090555668 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:28:11:729.018 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:11:729.031 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:11:729.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388091728769 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 59 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 220, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:28:11:730.261 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:11:734.258 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:11:734.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388091734231 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 530 in_bytes = 39077 out_tpkts = 590 out_bytes = 41036 exception_packets = 29 exception_packets_allowed = 15 raw_vhost_stats= [ name = vhost0 in_pkts = 530 in_bytes = 39077 out_pkts = 590 out_bytes = 41036 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 6.451e-16 total_out_bandwidth_utilization = 9.86624e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 119, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 182, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 418 in_bytes = 33812 out_pkts = 533 out_bytes = 39155 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-03-31 Mon 02:28:11:734.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:28:11:734.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:11:734.507 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388091734268 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:28:11:734.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:28:13:745.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:28:13:745.513 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:28:13:745.522 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:28:13:745.638 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Idle 2025-03-31 Mon 02:28:13:745.725 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:13:745.761 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388093745569 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:28:13:745.769 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:28:13:745.777 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:28:13:745.808 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:13:746.043 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388093745714 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 61 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 224, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:28:13:746.065 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:22:520.142 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:55708 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:28:22:561.927 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:55708 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:28:31:995.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53876 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:28:32:080.997 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53880 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:28:32:125.941 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53880 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:28:40:555.248 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:40:555.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388120555212 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:28:40:555.322 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:28:40:555.360 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:40:555.495 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388120555336 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 553 in_bytes = 40814 out_pkts = 613 out_bytes = 42665 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 479 out_bw_usage = 449 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:28:40:555.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:28:40:555.529 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:40:555.545 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388120555340 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:28:40:555.555 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:28:41:729.097 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:729.183 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121728894 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 229 dequeues = 229 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 269 dequeues = 269 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388093745636 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388120555527 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:28:41:729.209 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:41:729.218 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:729.439 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121729044 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 62 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 40, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 229, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:28:41:729.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:28:41:734.460 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:734.529 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121734431 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.275 five_min_avg = 0.40125 fifteen_min_avg = 0.39625 ] cpu_share = 0.239583 ] ] 2025-03-31 Mon 02:28:41:734.540 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:28:41:734.549 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:734.568 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121734447 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.239583 used_sys_mem = 4269468306 one_min_cpuload = 0.275 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:28:41:734.578 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:28:41:734.588 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:734.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121734477 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 553 in_bytes = 40814 out_tpkts = 613 out_bytes = 42665 exception_packets = 30 exception_packets_allowed = 16 raw_vhost_stats= [ name = vhost0 in_pkts = 553 in_bytes = 40814 out_pkts = 613 out_bytes = 42665 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 438 in_bytes = 35315 out_pkts = 556 out_bytes = 40892 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:28:41:734.820 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:28:41:734.831 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:28:41:734.877 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388121734501 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:28:41:734.886 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:28:43:745.806 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:28:43:746.069 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:28:43:746.195 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:43:746.234 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388123745903 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:28:43:746.245 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:28:43:746.256 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:28:43:746.778 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388123746179 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 64 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 40, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 235, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:28:43:746.817 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:10:556.815 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:10:556.895 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388150556775 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:29:10:556.906 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:29:10:556.914 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:10:556.975 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388150556898 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 560 in_bytes = 41288 out_pkts = 620 out_bytes = 43031 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:29:10:557.003 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:29:10:557.014 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:10:557.025 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388150556905 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:29:10:557.031 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:29:11:729.313 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:11:729.427 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388151729209 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 240 dequeues = 240 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 281 dequeues = 281 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388123745912 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388150557013 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:29:11:729.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:11:729.459 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:11:729.749 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388151729282 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 65 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 40, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 240, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:29:11:729.774 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:11:734.694 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:11:734.986 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388151734661 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 560 in_bytes = 41288 out_tpkts = 620 out_bytes = 43031 raw_vhost_stats= [ name = vhost0 in_pkts = 560 in_bytes = 41288 out_pkts = 620 out_bytes = 43031 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 1.25767e-15 total_out_bandwidth_utilization = 1.59378e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 232, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 294, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 442 in_bytes = 35555 out_pkts = 563 out_bytes = 41366 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:29:11:735.017 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:29:11:735.030 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:11:735.085 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388151734698 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:29:11:735.097 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:29:13:746.216 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:29:13:746.319 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:29:13:746.367 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:29:13:746.475 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:29:13:746.584 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:13:746.616 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388153746408 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:29:13:746.624 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:29:13:746.636 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:29:13:746.661 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:13:746.912 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388153746559 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 67 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 40, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 244, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:29:13:746.952 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:32:874.242 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:55574 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:29:40:558.639 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:40:558.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388180558600 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:29:40:559.518 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:29:40:559.546 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:40:559.629 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388180558780 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 567 in_bytes = 41762 out_pkts = 627 out_bytes = 43397 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:29:40:559.640 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:29:40:559.648 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:40:559.658 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388180559498 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:29:40:559.665 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:29:41:729.542 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:729.629 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388181729388 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 249 dequeues = 249 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 292 dequeues = 292 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388153746473 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388180559647 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:29:41:729.656 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:41:729.665 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:729.924 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388181729497 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 68 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 249, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:29:41:729.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:29:41:734.903 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:734.984 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388181734867 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.22625 five_min_avg = 0.36375 fifteen_min_avg = 0.3825 ] cpu_share = 0.239583 ] ] 2025-03-31 Mon 02:29:41:734.994 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:29:41:735.004 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:735.019 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388181734887 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.239583 used_sys_mem = 4269468306 one_min_cpuload = 0.22625 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:29:41:735.026 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:29:41:735.032 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:735.183 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388181734916 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 567 in_bytes = 41762 out_tpkts = 627 out_bytes = 43397 raw_vhost_stats= [ name = vhost0 in_pkts = 567 in_bytes = 41762 out_pkts = 627 out_bytes = 43397 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 446 in_bytes = 35795 out_pkts = 570 out_bytes = 41840 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:29:41:735.208 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:29:41:735.217 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:29:41:735.243 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388181734950 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:29:41:735.252 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:29:43:746.579 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:29:43:746.810 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:29:43:746.904 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:43:746.932 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388183746657 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:29:43:746.940 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:29:43:746.949 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:29:43:747.180 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388183746893 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 255, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:29:43:747.206 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:10:558.776 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:10:558.831 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388210558704 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:30:10:558.841 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:30:10:558.868 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:10:558.937 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388210558826 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 657 in_bytes = 49831 out_pkts = 719 out_bytes = 51470 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2151 out_bw_usage = 2152 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:30:10:558.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:30:10:558.975 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:10:558.988 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388210558829 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:30:10:558.995 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:30:11:729.785 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:11:729.896 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388211729655 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 260 dequeues = 260 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 304 dequeues = 304 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388183746663 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388210558973 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:30:11:729.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:11:729.935 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:11:730.152 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388211729740 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 260, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:30:11:730.168 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:11:735.141 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:11:735.435 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388211735108 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 657 in_bytes = 49831 out_tpkts = 719 out_bytes = 51470 exception_packets = 33 exception_packets_dropped = 15 exception_packets_allowed = 18 raw_vhost_stats= [ name = vhost0 in_pkts = 657 in_bytes = 49831 out_pkts = 719 out_bytes = 51470 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 387 out_bandwidth_usage = 436 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 5.91432e-15 total_out_bandwidth_utilization = 6.20164e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1091, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1144, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 535 in_bytes = 43742 out_pkts = 661 out_bytes = 49951 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:30:11:735.464 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:30:11:735.477 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:11:735.521 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388211735153 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:30:11:735.531 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:30:13:747.001 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:30:13:747.073 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:30:13:747.100 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:30:13:747.216 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Idle 2025-03-31 Mon 02:30:13:747.347 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:13:747.383 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388213747142 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:30:13:747.393 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:30:13:747.404 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:30:13:747.438 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:13:747.841 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388213747333 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 264, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:30:13:747.870 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:33:794.079 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:45058 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:30:40:558.900 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:40:558.959 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388240558861 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:30:40:558.971 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:30:40:559.074 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:40:559.182 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388240559051 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 690 in_bytes = 52658 out_pkts = 752 out_bytes = 54189 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 753 out_bw_usage = 725 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:30:40:559.196 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:30:40:559.208 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:40:559.218 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388240559056 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:30:40:559.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:30:41:730.026 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:730.119 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241729865 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 269 dequeues = 269 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 315 dequeues = 315 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388213747214 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388240559206 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:30:41:730.133 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:41:730.155 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:730.412 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241729982 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 269, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:30:41:730.432 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:30:41:735.374 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:735.457 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241735337 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.35375 five_min_avg = 0.39 fifteen_min_avg = 0.39125 ] cpu_share = 0.235417 ] ] 2025-03-31 Mon 02:30:41:735.468 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:30:41:735.480 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:735.495 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241735356 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.235417 used_sys_mem = 4269468306 one_min_cpuload = 0.35375 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:30:41:735.502 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:30:41:735.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:735.634 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241735388 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 690 in_bytes = 52658 out_tpkts = 752 out_bytes = 54189 raw_vhost_stats= [ name = vhost0 in_pkts = 690 in_bytes = 52658 out_pkts = 752 out_bytes = 54189 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 565 in_bytes = 46335 out_pkts = 694 out_bytes = 52778 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:30:41:735.656 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:30:41:735.664 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:30:41:735.691 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388241735414 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:30:41:735.701 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:30:43:747.356 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:30:43:747.590 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:30:43:747.698 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:43:747.743 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388243747436 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:30:43:747.751 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:30:43:747.759 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:30:43:748.037 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388243747684 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 275, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:30:43:748.076 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:10:559.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:10:559.104 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388270559000 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:31:10:559.115 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:31:10:559.162 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:10:559.259 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388270559136 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 698 in_bytes = 53202 out_pkts = 759 out_bytes = 54555 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 145 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:31:10:559.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:31:10:559.294 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:10:559.304 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388270559147 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:31:10:559.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:31:11:730.341 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:11:730.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388271730149 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 280 dequeues = 280 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 327 dequeues = 327 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388243747445 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388270559292 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:31:11:730.478 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:11:730.487 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:11:730.747 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388271730227 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 280, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:31:11:730.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:11:735.656 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:11:735.944 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388271735619 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 698 in_bytes = 53202 out_tpkts = 759 out_bytes = 54555 raw_vhost_stats= [ name = vhost0 in_pkts = 698 in_bytes = 53202 out_pkts = 759 out_bytes = 54555 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 2.04372e-15 total_out_bandwidth_utilization = 2.43403e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 377, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 449, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 569 in_bytes = 46575 out_pkts = 702 out_bytes = 53322 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:31:11:736.651 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:31:11:736.675 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:11:736.723 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388271735661 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:31:11:736.747 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:31:13:747.756 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:31:13:747.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:31:13:747.883 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:31:13:747.988 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:31:13:748.129 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:13:748.175 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388273747925 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:31:13:748.186 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:31:13:748.195 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:31:13:748.226 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:13:748.473 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388273748097 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 42, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 284, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:31:13:748.496 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:34:682.048 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:37752 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:31:40:560.783 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:40:560.870 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388300560752 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:31:40:560.882 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:31:40:560.922 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:40:561.049 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388300560892 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 705 in_bytes = 53676 out_pkts = 768 out_bytes = 55061 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 134 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:31:40:561.075 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:31:40:561.084 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:40:561.095 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388300560907 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:31:40:561.101 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:31:41:730.465 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:730.542 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388301730339 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 289 dequeues = 289 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 338 dequeues = 338 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388273747986 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388300561082 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:31:41:730.554 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:41:730.564 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:730.858 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388301730437 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 289, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:31:41:731.499 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:31:41:735.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:735.932 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388301735826 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.26875 five_min_avg = 0.36 fifteen_min_avg = 0.38125 ] cpu_share = 0.239583 ] ] 2025-03-31 Mon 02:31:41:735.943 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:31:41:735.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:735.966 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388301735841 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.239583 used_sys_mem = 4269468306 one_min_cpuload = 0.26875 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:31:41:735.973 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:31:41:735.980 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:736.106 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388301735870 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 705 in_bytes = 53676 out_tpkts = 768 out_bytes = 55061 raw_vhost_stats= [ name = vhost0 in_pkts = 705 in_bytes = 53676 out_pkts = 768 out_bytes = 55061 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 575 in_bytes = 46955 out_pkts = 709 out_bytes = 53796 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:31:41:736.123 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:31:41:736.130 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:31:41:736.156 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388301735903 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:31:41:736.163 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:31:43:748.176 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:31:43:748.401 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:31:43:748.528 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:43:748.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388303748255 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:31:43:748.567 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:31:43:748.590 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:31:43:748.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388303748515 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 295, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:31:43:748.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:10:563.487 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:10:563.548 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388330563448 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:32:10:563.559 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:32:10:563.657 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:10:563.719 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388330563648 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 711 in_bytes = 54108 out_pkts = 774 out_bytes = 55385 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 115 out_bw_usage = 86 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:32:10:563.772 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:32:10:563.781 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:10:563.792 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388330563657 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:32:10:563.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:32:11:730.706 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:11:730.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388331730590 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 300 dequeues = 300 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 350 dequeues = 350 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388303748261 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388330563780 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:32:11:730.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:11:730.857 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:11:731.102 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388331730660 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 300, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:32:11:731.120 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:11:736.094 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:11:736.339 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388331736056 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 711 in_bytes = 54108 out_tpkts = 774 out_bytes = 55385 raw_vhost_stats= [ name = vhost0 in_pkts = 711 in_bytes = 54108 out_pkts = 774 out_bytes = 55385 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 4.17418e-16 total_out_bandwidth_utilization = 6.50521e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 77, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 120, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 578 in_bytes = 47153 out_pkts = 715 out_bytes = 54228 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:32:11:736.366 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:32:11:736.378 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:11:736.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388331736105 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:32:11:736.414 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:32:13:748.539 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:32:13:748.613 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:32:13:748.626 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:32:13:748.711 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Idle 2025-03-31 Mon 02:32:13:748.805 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:13:748.833 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388333748661 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:32:13:748.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:32:13:748.860 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:32:13:748.889 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:13:749.153 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388333748794 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 304, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:32:13:749.181 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:35:582.117 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:45372 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:32:40:562.509 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:40:562.567 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388360562472 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:32:40:562.579 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:32:40:562.587 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:40:562.649 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388360562574 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 718 in_bytes = 54582 out_pkts = 781 out_bytes = 55751 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 130 out_bw_usage = 100 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:32:40:562.672 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:32:40:562.682 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:40:562.692 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388360562577 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:32:40:562.699 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:32:41:730.974 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:731.063 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388361730788 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 309 dequeues = 309 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 361 dequeues = 361 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388333748709 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388360562681 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:32:41:731.077 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:41:731.086 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:731.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388361730931 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 309, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:32:41:731.375 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:32:41:736.283 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:736.368 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388361736245 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.31625 five_min_avg = 0.35375 fifteen_min_avg = 0.3775 ] cpu_share = 0.235417 ] ] 2025-03-31 Mon 02:32:41:736.381 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:32:41:736.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:736.430 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388361736262 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.235417 used_sys_mem = 4269468306 one_min_cpuload = 0.31625 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:32:41:736.441 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:32:41:736.450 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:736.696 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388361736290 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 718 in_bytes = 54582 out_tpkts = 781 out_bytes = 55751 raw_vhost_stats= [ name = vhost0 in_pkts = 718 in_bytes = 54582 out_pkts = 781 out_bytes = 55751 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 582 in_bytes = 47393 out_pkts = 722 out_bytes = 54702 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:32:41:736.722 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:32:41:736.740 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:32:41:736.783 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388361736312 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:32:41:736.792 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:32:43:748.883 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:32:43:749.159 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:32:43:749.256 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:43:749.285 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388363748978 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:32:43:749.302 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:32:43:749.310 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:32:43:749.526 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388363749244 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 315, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:32:43:749.546 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:10:564.126 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:10:564.201 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388390564083 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:33:10:564.214 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:33:10:564.226 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:10:564.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388390564217 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 725 in_bytes = 55056 out_pkts = 788 out_bytes = 56117 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:33:10:564.331 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:33:10:564.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:10:564.355 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388390564223 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:33:10:564.362 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:33:11:731.213 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:11:731.320 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388391731088 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 320 dequeues = 320 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 373 dequeues = 373 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388363749000 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388390564341 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:33:11:731.334 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:11:731.342 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:11:731.553 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388391731160 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 320, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:33:11:731.585 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:11:736.523 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:11:736.721 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388391736491 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 725 in_bytes = 55056 out_tpkts = 788 out_bytes = 56117 raw_vhost_stats= [ name = vhost0 in_pkts = 725 in_bytes = 55056 out_pkts = 788 out_bytes = 56117 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 2 ] total_in_bandwidth_utilization = 3.46945e-16 total_out_bandwidth_utilization = 6.83047e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 64, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 126, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 586 in_bytes = 47633 out_pkts = 729 out_bytes = 55176 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 1 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ], ] ] ] 2025-03-31 Mon 02:33:11:736.760 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:33:11:736.773 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:11:736.801 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388391736538 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:33:11:736.808 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:33:13:749.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:33:13:749.401 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:33:13:749.438 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:33:13:749.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:33:13:749.629 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:13:749.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388393749470 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:33:13:749.668 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:33:13:749.675 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:33:13:749.737 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:13:749.950 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388393749618 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 324, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:33:13:749.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:21:276.542 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34378 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:33:21:321.867 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34378 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:33:35:305.128 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38238 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:33:35:350.189 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38238 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:33:36:470.047 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:38250 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:33:40:565.783 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:40:565.846 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388420565719 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:33:40:565.856 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:33:40:565.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:40:565.929 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388420565860 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 746 in_bytes = 56572 out_pkts = 807 out_bytes = 58032 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 404 out_bw_usage = 510 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 4 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 5 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:33:40:565.951 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:33:40:565.961 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:40:565.971 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388420565865 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:33:40:565.978 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:33:41:731.445 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:731.549 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388421731286 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 329 dequeues = 329 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 384 dequeues = 384 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388393749532 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388420565959 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:33:41:731.564 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:41:731.576 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:731.821 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388421731391 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 329, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:33:41:731.852 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:41:736.804 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:736.900 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388421736703 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.26625 five_min_avg = 0.32875 fifteen_min_avg = 0.36625 ] cpu_share = 0.260417 ] ] 2025-03-31 Mon 02:33:41:736.912 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:33:41:736.924 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:736.945 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388421736789 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.260417 used_sys_mem = 4269468306 one_min_cpuload = 0.26625 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:33:41:736.953 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:33:41:736.962 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:737.169 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388421736831 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 746 in_bytes = 56572 out_tpkts = 807 out_bytes = 58032 exception_packets = 35 exception_packets_allowed = 20 raw_vhost_stats= [ name = vhost0 in_pkts = 746 in_bytes = 56572 out_pkts = 807 out_bytes = 58032 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 4 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 5 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 4 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 5 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 602 in_bytes = 49422 out_pkts = 751 out_bytes = 56734 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 4 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 4 ], ] ] ] 2025-03-31 Mon 02:33:41:737.192 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:33:41:737.200 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:33:41:737.226 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388421736872 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:33:41:737.236 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:33:43:749.660 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:33:43:749.879 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:33:43:750.014 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:43:750.048 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388423749753 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:33:43:750.056 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:33:43:750.064 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:33:43:750.278 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388423749997 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 335, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:33:43:750.297 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:33:43:893.605 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40672 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:33:43:937.925 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40672 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:33:54:919.794 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40700 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-03-31 Mon 02:33:54:961.909 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40700 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-03-31 Mon 02:34:10:567.407 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:10:567.470 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388450567371 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:34:10:567.494 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:34:10:567.508 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:10:567.567 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388450567502 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 764 in_bytes = 57934 out_pkts = 822 out_bytes = 59064 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 363 out_bw_usage = 275 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 7 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 8 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:34:10:567.578 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:34:10:567.586 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:10:567.597 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388450567507 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:34:10:567.606 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:34:11:731.686 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:11:731.841 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388451731550 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 340 dequeues = 340 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 396 dequeues = 396 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388423749761 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388450567584 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:34:11:731.871 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:34:11:731.884 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:11:732.139 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388451731637 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 95 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 53, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 340, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:34:11:732.162 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:34:11:737.075 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:11:737.271 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388451737041 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 764 in_bytes = 57934 out_tpkts = 822 out_bytes = 59064 exception_packets = 36 exception_packets_allowed = 21 raw_vhost_stats= [ name = vhost0 in_pkts = 764 in_bytes = 57934 out_pkts = 822 out_bytes = 59064 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 7 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 8 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 7 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 8 ] total_in_bandwidth_utilization = 1.94614e-15 total_out_bandwidth_utilization = 2.10877e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 359, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 389, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 614 in_bytes = 50328 out_pkts = 769 out_bytes = 58096 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 7 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 7 ], ] ] ] 2025-03-31 Mon 02:34:11:737.298 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:34:11:737.308 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:11:737.343 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388451737093 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:34:11:737.353 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:34:13:750.033 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:34:13:750.159 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:34:13:750.195 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:34:13:750.286 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Idle 2025-03-31 Mon 02:34:13:750.376 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:13:750.403 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388453750228 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:34:13:750.411 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:34:13:750.418 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:34:13:750.449 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:13:750.671 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388453750364 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 53, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 344, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:34:13:750.697 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:34:37:450.250 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:56860 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:34:40:567.777 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:40:567.828 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388480567715 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:34:40:567.838 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:34:40:567.857 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:40:568.017 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388480567764 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 857 in_bytes = 66124 out_pkts = 939 out_bytes = 68466 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2184 out_bw_usage = 2507 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 10 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:34:40:568.032 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:34:40:568.041 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:40:568.052 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388480567768 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:34:40:568.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:34:41:731.921 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:732.015 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388481731768 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 349 dequeues = 349 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 407 dequeues = 407 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388453750285 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388480568040 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:34:41:732.041 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:34:41:732.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:732.313 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388481731874 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 98 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 43 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 349, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:34:41:732.333 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:34:41:737.267 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:737.367 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388481737225 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.70125 five_min_avg = 0.4175 fifteen_min_avg = 0.39375 ] cpu_share = 0.285369 ] ] 2025-03-31 Mon 02:34:41:737.380 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:34:41:737.391 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:737.410 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388481737245 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.285369 used_sys_mem = 4269468306 one_min_cpuload = 0.70125 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:34:41:737.419 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:34:41:737.443 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:737.625 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388481737280 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 857 in_bytes = 66124 out_tpkts = 939 out_bytes = 68466 exception_packets = 37 exception_packets_allowed = 22 raw_vhost_stats= [ name = vhost0 in_pkts = 857 in_bytes = 66124 out_pkts = 939 out_bytes = 68466 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 10 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 10 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 728 in_bytes = 59604 out_pkts = 862 out_bytes = 66286 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 10 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 10 ], ] ] ] 2025-03-31 Mon 02:34:41:737.651 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:34:41:737.665 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:34:41:737.701 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388481737308 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:34:41:737.714 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:34:43:750.438 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:34:43:750.662 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:34:43:750.784 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:43:750.815 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388483750526 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:34:43:750.823 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:34:43:750.831 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:34:43:751.059 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388483750770 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 100 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 43 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 355, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:34:43:751.090 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:10:569.611 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:10:569.658 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388510569494 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:35:10:569.669 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:35:10:569.678 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:10:569.781 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388510569584 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 881 in_bytes = 71143 out_pkts = 955 out_bytes = 69703 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 1338 out_bw_usage = 329 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:35:10:569.795 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:35:10:569.803 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:10:569.815 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388510569587 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:35:10:569.821 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:35:11:732.144 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:11:732.255 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388511732040 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 360 dequeues = 360 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 419 dequeues = 419 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388483750534 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388510569802 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:35:11:732.282 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:11:732.290 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:11:732.498 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388511732116 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 101 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 43 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 360, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:35:11:732.515 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:11:737.504 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:11:737.766 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388511737469 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 881 in_bytes = 71143 out_tpkts = 955 out_bytes = 69703 exception_packets = 39 exception_packets_allowed = 24 raw_vhost_stats= [ name = vhost0 in_pkts = 881 in_bytes = 71143 out_pkts = 955 out_bytes = 69703 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 452 out_bandwidth_usage = 570 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] total_in_bandwidth_utilization = 7.50268e-15 total_out_bandwidth_utilization = 9.57351e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1384, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1766, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 741 in_bytes = 60715 out_pkts = 887 out_bytes = 71347 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ], ] ] ] 2025-03-31 Mon 02:35:11:737.793 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:35:11:737.804 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:11:737.832 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388511737521 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:35:11:737.839 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:35:13:750.812 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:35:13:750.923 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:35:13:750.962 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:35:13:751.099 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Idle 2025-03-31 Mon 02:35:13:751.222 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:13:751.249 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388513751036 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:35:13:751.257 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:35:13:751.265 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:35:13:751.304 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:13:751.533 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388513751203 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 103 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 43 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 54, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 364, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:35:13:751.557 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:38:351.670 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386802280768, Pid 1197922]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:43020 > SSL Handshake failed due to error: 336130315 category: asio.ssl message: wrong version number src/contrail-common/io/ssl_server.cc 96 2025-03-31 Mon 02:35:40:569.645 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:40:569.743 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388540569605 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:35:40:569.755 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:35:40:570.042 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:40:570.149 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388540570019 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 890 in_bytes = 72134 out_pkts = 963 out_bytes = 70111 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 264 out_bw_usage = 108 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:35:40:570.176 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:35:40:570.186 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:40:570.197 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388540570029 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:35:40:570.204 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:35:41:732.405 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:732.492 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388541732243 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 369 dequeues = 369 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 430 dequeues = 430 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1743388513751096 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388540570184 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:35:41:732.506 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:41:732.515 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:732.787 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: SANDESH: Send FAILED: 1743388541732360 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 104 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 369, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:35:41:732.807 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386640574208, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:35:41:737.773 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:737.866 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388541737739 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602560 used = 0 free = 1469799982 buffers = 32686 cached = 25466304 node_type = ] meminfo= [ virt = 32686 peakvirt = 1455765912 res = 32686 ] cpuload= [ one_min_avg = 0.5425 five_min_avg = 0.41375 fifteen_min_avg = 0.3925 ] cpu_share = 0.4 ] ] 2025-03-31 Mon 02:35:41:737.881 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:35:41:737.893 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:737.922 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388541737760 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 cpu_info= [ [ [ mem_virt = 32686 cpu_share = 0.4 used_sys_mem = 4269468306 one_min_cpuload = 0.5425 mem_res = 32686 ], ] ] ] 2025-03-31 Mon 02:35:41:737.932 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-03-31 Mon 02:35:41:737.942 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:738.089 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388541737794 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 890 in_bytes = 72134 out_tpkts = 963 out_bytes = 70111 exception_packets = 40 exception_packets_allowed = 25 raw_vhost_stats= [ name = vhost0 in_pkts = 890 in_bytes = 72134 out_pkts = 963 out_bytes = 70111 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 746 in_bytes = 60997 out_pkts = 896 out_bytes = 72338 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ], ] ] ] 2025-03-31 Mon 02:35:41:738.112 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:35:41:738.123 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:35:41:738.162 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388541737861 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:35:41:738.169 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:35:43:751.677 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-03-31 Mon 02:35:43:752.534 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Connect : Start Connect timer 10.0.0.17:8086 2025-03-31 Mon 02:35:43:752.652 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:43:752.678 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388543752332 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:35:43:752.686 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:35:43:752.693 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:35:43:752.918 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: SANDESH: Send FAILED: 1743388543752639 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 106 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 375, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:35:43:752.947 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386644772608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:36:10:571.252 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:10:571.311 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: SANDESH: Send FAILED: 1743388570571219 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-03-31 Mon 02:36:10:571.321 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386623780608, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-03-31 Mon 02:36:10:571.388 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:10:571.480 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388570571367 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 927 in_bytes = 75153 out_pkts = 1000 out_bytes = 73078 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 805 out_bw_usage = 791 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] policy_rules= [ [ ] ] ] 2025-03-31 Mon 02:36:10:571.507 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-03-31 Mon 02:36:10:571.516 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:10:571.527 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: SANDESH: Send FAILED: 1743388570571372 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-494-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-03-31 Mon 02:36:10:571.533 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386653169408, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-03-31 Mon 02:36:11:732.591 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:11:732.704 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388571732476 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 380 dequeues = 380 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 442 dequeues = 442 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1743388543752349 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388570571514 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-03-31 Mon 02:36:11:732.754 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:36:11:732.764 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:11:733.016 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: SANDESH: Send FAILED: 1743388571732549 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 107 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 380, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:36:11:733.038 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386627979008, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-03-31 Mon 02:36:11:738.011 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:11:738.190 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388571737979 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 in_tpkts = 927 in_bytes = 75153 out_tpkts = 1000 out_bytes = 73078 exception_packets = 41 exception_packets_allowed = 26 raw_vhost_stats= [ name = vhost0 in_pkts = 927 in_bytes = 75153 out_pkts = 1000 out_bytes = 73078 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 12 ] total_in_bandwidth_utilization = 2.25514e-15 total_out_bandwidth_utilization = 2.89482e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 416, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 534, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 780 in_bytes = 63838 out_pkts = 933 out_bytes = 75357 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 11 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 11 ], ] ] ] 2025-03-31 Mon 02:36:11:738.215 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-03-31 Mon 02:36:11:738.225 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Processing scm::EvSandeshSend in state Connect 2025-03-31 Mon 02:36:11:738.252 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: SANDESH: Send FAILED: 1743388571738023 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 9 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-03-31 Mon 02:36:11:738.258 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386636375808, Pid 1197922]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-03-31 Mon 02:36:13:752.692 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:36:13:752.799 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvConnectTimerExpired in state Connect 2025-03-31 Mon 02:36:13:752.812 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Connect : EvConnectTimerExpired 2025-03-31 Mon 02:36:13:752.897 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Idle 2025-03-31 Mon 02:36:13:753.007 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:36:13:753.034 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388573752842 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:36:13:753.053 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-03-31 Mon 02:36:13:753.060 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvTcpDeleteSession in state Idle 2025-03-31 Mon 02:36:13:753.084 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: Processing scm::EvSandeshSend in state Idle 2025-03-31 Mon 02:36:13:753.296 UTC cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140386632177408, Pid 1197922]: SANDESH: Send FAILED: 1743388573752972 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1197922 http_port = 8085 start_time = 1743387581724103 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 109 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 384, _iter59->first = sent _iter59->second = 0, ] ] ]