2025-02-13 Thu 02:18:15:121.059 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: vhost interface name : vhost0 2025-02-13 Thu 02:18:15:121.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: vhost IP Address : 10.0.0.17/24 2025-02-13 Thu 02:18:15:121.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: vhost gateway : 0.0.0.0 2025-02-13 Thu 02:18:15:121.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Ethernet ports : ens3 2025-02-13 Thu 02:18:15:121.269 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Loopback IP : 0.0.0.0 2025-02-13 Thu 02:18:15:121.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Servers : 10.0.0.17:5269 2025-02-13 Thu 02:18:15:121.282 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Authentication : 1 2025-02-13 Thu 02:18:15:121.287 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:15:121.292 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:15:121.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:15:121.303 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Cluster-Name : 2025-02-13 Thu 02:18:15:121.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: DNS Servers : 10.0.0.17:53 2025-02-13 Thu 02:18:15:121.314 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: DNS client port : 8098 2025-02-13 Thu 02:18:15:121.319 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: DNS timeout : 3000 2025-02-13 Thu 02:18:15:121.324 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: DNS max retries : 2 2025-02-13 Thu 02:18:15:121.329 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Dns Authentication : 1 2025-02-13 Thu 02:18:15:121.334 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:15:121.339 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:15:121.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:15:121.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TSN Servers : [] 2025-02-13 Thu 02:18:15:121.354 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: COLLECTOR Servers : 10.0.0.17:8086 2025-02-13 Thu 02:18:15:121.359 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Tunnel-Type : MPLSoGRE 2025-02-13 Thu 02:18:15:121.364 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Metadata-Proxy Shared Secret: c9392c64-153a-4aa0-b00d-4fa0ced2fcc1 2025-02-13 Thu 02:18:15:121.369 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Metadata-Proxy Port : 8097 2025-02-13 Thu 02:18:15:121.374 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Metadata-Proxy SSL Flag : 0 2025-02-13 Thu 02:18:15:121.389 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Max Vm Flows : 100 2025-02-13 Thu 02:18:15:121.399 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Linklocal Max System Flows : 16384 2025-02-13 Thu 02:18:15:121.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Linklocal Max Vm Flows : 16384 2025-02-13 Thu 02:18:15:121.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow cache timeout : 0 2025-02-13 Thu 02:18:15:121.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Stale Interface cleanup timeout : 60 2025-02-13 Thu 02:18:15:121.419 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow thread count : 1 2025-02-13 Thu 02:18:15:121.424 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow latency limit : 0 2025-02-13 Thu 02:18:15:121.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow index-mgr sm log count : 0 2025-02-13 Thu 02:18:15:121.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow add-tokens : 50 2025-02-13 Thu 02:18:15:121.439 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow ksync-tokens : 25 2025-02-13 Thu 02:18:15:121.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow del-tokens : 16 2025-02-13 Thu 02:18:15:121.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow update-tokens : 16 2025-02-13 Thu 02:18:15:121.454 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Pin flow netlink task to CPU: 2025-02-13 Thu 02:18:15:121.459 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Maximum sessions : 100 2025-02-13 Thu 02:18:15:121.464 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Maximum session aggregates : 8 2025-02-13 Thu 02:18:15:121.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Maximum session endpoints : 5 2025-02-13 Thu 02:18:15:121.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Fabric SNAT hash table size : 4096 2025-02-13 Thu 02:18:15:121.478 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Flow excluding Router ID in hash :0 2025-02-13 Thu 02:18:15:121.483 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Agent Mode : Vrouter 2025-02-13 Thu 02:18:15:121.488 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Gateway Mode : None 2025-02-13 Thu 02:18:15:121.493 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: DHCP Relay Mode : 0 2025-02-13 Thu 02:18:15:121.498 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance netns cmd : /usr/local/bin/opencontrail-vrouter-netns 2025-02-13 Thu 02:18:15:121.503 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance docker cmd : /usr/local/bin/opencontrail-vrouter-docker 2025-02-13 Thu 02:18:15:121.507 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance workers : 0 2025-02-13 Thu 02:18:15:121.512 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance timeout : 0 2025-02-13 Thu 02:18:15:121.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance lb ssl : 2025-02-13 Thu 02:18:15:121.522 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Service instance lbaas auth : 2025-02-13 Thu 02:18:15:121.527 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Bgp as a service port range : 50000-50512 2025-02-13 Thu 02:18:15:121.535 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Services queue limit : 1024 2025-02-13 Thu 02:18:15:121.540 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: BGPAAS max shared sessions for service port : 4 2025-02-13 Thu 02:18:15:121.545 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:15:121.550 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:15:121.554 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:15:121.559 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Sandesh SSL Enable : 1 2025-02-13 Thu 02:18:15:121.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Introspect SSL Enable : 1 2025-02-13 Thu 02:18:15:121.569 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Hypervisor mode : kvm 2025-02-13 Thu 02:18:15:121.680 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Ethernet Port Encap Type : 2025-02-13 Thu 02:18:15:121.687 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: Platform mode : Vrouter on host linux kernel 2025-02-13 Thu 02:18:15:121.979 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: Logging: DISABLED -> ENABLED 2025-02-13 Thu 02:18:15:122.050 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2025-02-13 Thu 02:18:15:122.051 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2025-02-13 00:26:33.389417","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-7769-1.novalocal","build-id":"unknown","build-number":"unknown"}]} src/contrail-common/base/misc_utils.cc 41 2025-02-13 Thu 02:18:15:145.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2025-02-13 Thu 02:18:15:145.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:15:145.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:15:145.486 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.491 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2025-02-13 Thu 02:18:15:145.497 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2025-02-13 Thu 02:18:15:145.502 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2025-02-13 Thu 02:18:15:145.509 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.523 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2025-02-13 Thu 02:18:15:145.528 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:15:145.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2025-02-13 Thu 02:18:15:145.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:15:145.546 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.551 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2025-02-13 Thu 02:18:15:145.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2025-02-13 Thu 02:18:15:145.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2025-02-13 Thu 02:18:15:145.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.573 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2025-02-13 Thu 02:18:15:145.580 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2025-02-13 Thu 02:18:15:145.585 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2025-02-13 Thu 02:18:15:145.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2025-02-13 Thu 02:18:15:145.602 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2025-02-13 Thu 02:18:15:145.607 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2025-02-13 Thu 02:18:15:145.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2025-02-13 Thu 02:18:15:145.619 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2025-02-13 Thu 02:18:15:145.625 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:15:145.630 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2025-02-13 Thu 02:18:15:145.636 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2025-02-13 Thu 02:18:15:145.644 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2025-02-13 Thu 02:18:15:145.650 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for routing-instance 2025-02-13 Thu 02:18:15:145.656 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2025-02-13 Thu 02:18:15:145.661 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:15:145.667 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2025-02-13 Thu 02:18:15:145.672 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:15:145.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:15:145.684 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for security-group 2025-02-13 Thu 02:18:15:145.689 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-group : security-group-access-control-list -> self 2025-02-13 Thu 02:18:15:145.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2025-02-13 Thu 02:18:15:145.700 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2025-02-13 Thu 02:18:15:145.707 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for tag 2025-02-13 Thu 02:18:15:145.712 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> self 2025-02-13 Thu 02:18:15:145.718 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:145.723 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2025-02-13 Thu 02:18:15:145.729 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2025-02-13 Thu 02:18:15:145.735 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.740 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:15:145.746 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:145.751 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:145.756 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:15:145.764 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:15:145.771 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.776 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-02-13 Thu 02:18:15:145.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:15:145.786 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2025-02-13 Thu 02:18:15:145.792 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.797 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-02-13 Thu 02:18:15:145.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:15:145.808 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2025-02-13 Thu 02:18:15:145.814 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.819 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:15:145.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:145.829 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:145.835 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:15:145.840 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:15:145.846 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.851 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2025-02-13 Thu 02:18:15:145.857 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:15:145.862 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:15:145.868 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:15:145.876 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2025-02-13 Thu 02:18:15:145.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.909 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2025-02-13 Thu 02:18:15:145.915 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2025-02-13 Thu 02:18:15:145.920 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2025-02-13 Thu 02:18:15:145.926 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.931 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:145.937 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:145.943 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:145.949 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2025-02-13 Thu 02:18:15:145.955 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2025-02-13 Thu 02:18:15:145.960 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2025-02-13 Thu 02:18:15:145.967 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:145.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:145.978 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:145.983 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:145.990 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-02-13 Thu 02:18:15:145.995 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:15:146.047 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:15:146.053 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:15:146.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:15:146.069 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.085 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.090 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-02-13 Thu 02:18:15:146.106 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:15:146.112 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:15:146.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:15:146.122 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:15:146.129 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.134 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.139 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.145 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.150 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2025-02-13 Thu 02:18:15:146.156 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2025-02-13 Thu 02:18:15:146.162 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2025-02-13 Thu 02:18:15:146.167 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.172 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.177 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:15:146.192 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2025-02-13 Thu 02:18:15:146.218 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2025-02-13 Thu 02:18:15:146.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2025-02-13 Thu 02:18:15:146.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.235 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.241 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:15:146.253 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.258 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.265 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.272 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-02-13 Thu 02:18:15:146.278 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:15:146.284 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:15:146.291 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.296 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.302 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.307 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:15:146.312 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:15:146.317 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:15:146.327 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.338 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.350 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2025-02-13 Thu 02:18:15:146.355 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2025-02-13 Thu 02:18:15:146.360 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2025-02-13 Thu 02:18:15:146.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2025-02-13 Thu 02:18:15:146.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.376 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.382 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.387 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.393 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2025-02-13 Thu 02:18:15:146.398 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2025-02-13 Thu 02:18:15:146.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.409 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2025-02-13 Thu 02:18:15:146.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2025-02-13 Thu 02:18:15:146.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2025-02-13 Thu 02:18:15:146.426 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2025-02-13 Thu 02:18:15:146.437 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2025-02-13 Thu 02:18:15:146.445 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2025-02-13 Thu 02:18:15:146.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.457 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-02-13 Thu 02:18:15:146.463 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.474 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.485 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:15:146.491 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-02-13 Thu 02:18:15:146.496 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.501 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-02-13 Thu 02:18:15:146.506 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.511 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.516 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:15:146.522 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2025-02-13 Thu 02:18:15:146.528 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.533 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2025-02-13 Thu 02:18:15:146.538 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2025-02-13 Thu 02:18:15:146.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2025-02-13 Thu 02:18:15:146.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.563 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.579 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:146.586 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:15:146.597 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:146.602 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:146.608 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> virtual-network-tag 2025-02-13 Thu 02:18:15:146.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2025-02-13 Thu 02:18:15:146.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.623 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.633 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.638 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.643 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:146.650 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-02-13 Thu 02:18:15:146.663 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:15:146.669 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:15:146.674 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-02-13 Thu 02:18:15:146.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-02-13 Thu 02:18:15:146.684 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.689 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.694 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.704 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.709 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:146.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-02-13 Thu 02:18:15:146.726 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:15:146.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:15:146.737 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:15:146.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:15:146.747 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-02-13 Thu 02:18:15:146.752 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-02-13 Thu 02:18:15:146.757 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.762 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.771 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.776 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.786 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:146.793 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.798 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:146.804 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap project : self -> project-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.809 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2025-02-13 Thu 02:18:15:146.821 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> project-tag 2025-02-13 Thu 02:18:15:146.827 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2025-02-13 Thu 02:18:15:146.832 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.837 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:15:146.843 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.851 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.858 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:146.865 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:146.874 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.882 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2025-02-13 Thu 02:18:15:146.890 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-02-13 Thu 02:18:15:146.898 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-02-13 Thu 02:18:15:146.906 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-02-13 Thu 02:18:15:146.919 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2025-02-13 Thu 02:18:15:146.925 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:15:146.931 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2025-02-13 Thu 02:18:15:146.936 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router : self -> logical-router-interface 2025-02-13 Thu 02:18:15:146.942 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-router : logical-router-interface -> nil 2025-02-13 Thu 02:18:15:146.948 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for physical-interface 2025-02-13 Thu 02:18:15:146.953 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2025-02-13 Thu 02:18:15:146.958 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:15:146.964 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:15:146.969 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for physical-interface 2025-02-13 Thu 02:18:15:146.974 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2025-02-13 Thu 02:18:15:146.981 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:15:146.986 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:15:146.992 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:15:146.997 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2025-02-13 Thu 02:18:15:147.003 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:15:147.008 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2025-02-13 Thu 02:18:15:147.014 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2025-02-13 Thu 02:18:15:147.019 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-02-13 Thu 02:18:15:147.037 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:15:147.046 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:15:147.052 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:15:147.057 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2025-02-13 Thu 02:18:15:147.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2025-02-13 Thu 02:18:15:147.068 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:15:147.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:15:147.078 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2025-02-13 Thu 02:18:15:147.084 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2025-02-13 Thu 02:18:15:147.089 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2025-02-13 Thu 02:18:15:147.095 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for physical-router 2025-02-13 Thu 02:18:15:147.100 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2025-02-13 Thu 02:18:15:147.105 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:15:147.110 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:15:147.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for service-health-check 2025-02-13 Thu 02:18:15:147.123 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-02-13 Thu 02:18:15:147.128 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-02-13 Thu 02:18:15:147.134 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2025-02-13 Thu 02:18:15:147.139 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:147.144 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:15:147.150 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:15:147.157 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for service-health-check 2025-02-13 Thu 02:18:15:147.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-02-13 Thu 02:18:15:147.170 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-02-13 Thu 02:18:15:147.175 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2025-02-13 Thu 02:18:15:147.180 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-02-13 Thu 02:18:15:147.185 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-02-13 Thu 02:18:15:147.191 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-02-13 Thu 02:18:15:147.196 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2025-02-13 Thu 02:18:15:147.201 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:15:147.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:15:147.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:15:147.218 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for qos-config 2025-02-13 Thu 02:18:15:147.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2025-02-13 Thu 02:18:15:147.229 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2025-02-13 Thu 02:18:15:147.237 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for forwarding-class 2025-02-13 Thu 02:18:15:147.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2025-02-13 Thu 02:18:15:147.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2025-02-13 Thu 02:18:15:147.269 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2025-02-13 Thu 02:18:15:147.276 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:15:147.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2025-02-13 Thu 02:18:15:147.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2025-02-13 Thu 02:18:15:147.305 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2025-02-13 Thu 02:18:15:147.318 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:15:147.326 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2025-02-13 Thu 02:18:15:147.331 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2025-02-13 Thu 02:18:15:147.337 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2025-02-13 Thu 02:18:15:147.343 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:15:147.348 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2025-02-13 Thu 02:18:15:147.354 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2025-02-13 Thu 02:18:15:147.360 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2025-02-13 Thu 02:18:15:147.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2025-02-13 Thu 02:18:15:147.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:15:147.390 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2025-02-13 Thu 02:18:15:147.396 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2025-02-13 Thu 02:18:15:147.402 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2025-02-13 Thu 02:18:15:147.409 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2025-02-13 Thu 02:18:15:147.423 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for bridge-domain 2025-02-13 Thu 02:18:15:147.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-02-13 Thu 02:18:15:147.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:15:147.440 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:15:147.446 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:15:147.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:15:147.460 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for bridge-domain 2025-02-13 Thu 02:18:15:147.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-02-13 Thu 02:18:15:147.470 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:15:147.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-02-13 Thu 02:18:15:147.482 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for application-policy-set 2025-02-13 Thu 02:18:15:147.487 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2025-02-13 Thu 02:18:15:147.493 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:147.498 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:147.503 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:15:147.508 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:15:147.515 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:15:147.520 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:15:147.527 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.532 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.542 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2025-02-13 Thu 02:18:15:147.554 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2025-02-13 Thu 02:18:15:147.560 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:15:147.565 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:15:147.571 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.579 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.589 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.595 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2025-02-13 Thu 02:18:15:147.600 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2025-02-13 Thu 02:18:15:147.606 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> address-group-tag 2025-02-13 Thu 02:18:15:147.611 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : address-group-tag -> nil 2025-02-13 Thu 02:18:15:147.617 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:15:147.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:15:147.627 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.632 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.637 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.643 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : self -> firewall-rule-tag 2025-02-13 Thu 02:18:15:147.654 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap tag : firewall-rule-tag -> nil 2025-02-13 Thu 02:18:15:147.659 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:15:147.664 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:15:147.669 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.674 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:15:147.684 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2025-02-13 Thu 02:18:15:293.346 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.17:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:15:294.467 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Current receive sock buffer size is 262144 2025-02-13 Thu 02:18:15:294.565 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987826345728, Pid 1177849]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2025-02-13 Thu 02:18:15:295.634 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Vrouter family is 34 2025-02-13 Thu 02:18:15:295.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Flow table size : 20185088 2025-02-13 Thu 02:18:15:295.989 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Flow table size : 161218560 2025-02-13 Thu 02:18:15:296.009 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2025-02-13 Thu 02:18:15:296.055 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2025-02-13 Thu 02:18:15:296.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7f5172a00000 2025-02-13 Thu 02:18:15:296.768 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2025-02-13 Thu 02:18:15:296.778 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7f5169000000 2025-02-13 Thu 02:18:15:296.807 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Sending Huge Page configuration to VROUTER 2025-02-13 Thu 02:18:15:352.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Huge pages set successfully 2025-02-13 Thu 02:18:15:366.978 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Flow table size : 161218560 2025-02-13 Thu 02:18:15:367.187 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Flow table size : 20185088 2025-02-13 Thu 02:18:15:453.914 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: ROLE : Generator 2025-02-13 Thu 02:18:15:453.952 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: MODULE : contrail-vrouter-agent 2025-02-13 Thu 02:18:15:453.959 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-473-1 2025-02-13 Thu 02:18:15:453.964 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: NODE TYPE : Compute 2025-02-13 Thu 02:18:15:453.969 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: INSTANCE ID : 0 2025-02-13 Thu 02:18:15:453.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: HTTP SERVER PORT : 8085 2025-02-13 Thu 02:18:15:453.980 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: System Log Send Rate Limit: 0 -> 0 2025-02-13 Thu 02:18:15:455.430 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: HTTP Introspect Init 2025-02-13 Thu 02:18:15:455.748 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:15:455.822 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Sandesh Http Server Port: 8085 2025-02-13 Thu 02:18:15:455.892 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1177109.http_port 2025-02-13 Thu 02:18:15:455.899 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: CONNECT TO COLLECTOR: 1 2025-02-13 Thu 02:18:15:456.013 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Idle 2025-02-13 Thu 02:18:15:456.883 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: No Client: 1739413095456124 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:15:457.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvStart in state Idle 2025-02-13 Thu 02:18:15:458.508 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Disconnect 2025-02-13 Thu 02:18:15:458.947 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvCollectorUpdate in state Disconnect 2025-02-13 Thu 02:18:15:459.248 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:15:459.304 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:459.329 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095458487 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:15:459.345 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:15:459.353 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:460.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2025-02-13 Thu 02:18:15:460.548 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2025-02-13 Thu 02:18:15:460.593 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2025-02-13 Thu 02:18:15:460.599 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2025-02-13 Thu 02:18:15:460.633 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2025-02-13 Thu 02:18:15:460.642 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vethcbdd4c17 flags 69699 2025-02-13 Thu 02:18:15:460.673 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth48710ba3 flags 69699 2025-02-13 Thu 02:18:15:460.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth42eea4cc flags 69699 2025-02-13 Thu 02:18:15:460.710 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vethfc72e571 flags 69699 2025-02-13 Thu 02:18:15:460.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth74295baa flags 69699 2025-02-13 Thu 02:18:15:460.748 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth0c8bdc3e flags 69699 2025-02-13 Thu 02:18:15:460.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vetheea8ac85 flags 69699 2025-02-13 Thu 02:18:15:460.785 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vethfa20855f flags 69699 2025-02-13 Thu 02:18:15:460.791 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth772e951b flags 69699 2025-02-13 Thu 02:18:15:460.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vethc874d1a9 flags 69699 2025-02-13 Thu 02:18:15:460.826 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface veth5396329c flags 69699 2025-02-13 Thu 02:18:15:460.853 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vethe535c679 flags 69699 2025-02-13 Thu 02:18:15:460.859 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2025-02-13 Thu 02:18:15:460.882 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2025-02-13 Thu 02:18:15:460.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2025-02-13 Thu 02:18:15:460.900 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2025-02-13 Thu 02:18:15:460.912 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2025-02-13 Thu 02:18:15:460.917 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2025-02-13 Thu 02:18:15:460.970 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface lo 2025-02-13 Thu 02:18:15:460.981 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-02-13 Thu 02:18:15:460.991 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2025-02-13 Thu 02:18:15:461.001 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface docker0 2025-02-13 Thu 02:18:15:461.010 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-02-13 Thu 02:18:15:461.044 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface lo 2025-02-13 Thu 02:18:15:461.064 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-02-13 Thu 02:18:15:461.078 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface ftun0 2025-02-13 Thu 02:18:15:461.091 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-02-13 Thu 02:18:15:461.101 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink address message RTM_NEWADDR for interface pkt0 2025-02-13 Thu 02:18:15:461.130 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095458710 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-02-13 Thu 02:18:15:461.161 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-02-13 Thu 02:18:15:461.163 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:461.216 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.219 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095458764 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-02-13 Thu 02:18:15:461.254 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-02-13 Thu 02:18:15:461.244 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:461.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.299 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.311 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 10.200.97.0/24 Interface lxdbr0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.328 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.339 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.350 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2025-02-13 Thu 02:18:15:461.356 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:15:461.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095458932 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:15:461.391 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:15:461.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:15:461.407 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.416 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:15:461.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:15:461.453 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:15:461.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:15:461.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.484 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.489 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.499 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:15:461.504 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:15:461.514 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:15:461.402 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:461.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:15:461.554 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:15:461.559 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:15:461.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:15:461.569 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:15:461.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:15:461.579 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:15:461.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:15:461.589 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:15:461.594 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:15:461.687 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095458990 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:15:461.704 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:15:461.714 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:461.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095459010 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:15:461.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:15:461.755 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:461.821 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: SANDESH: Send FAILED: 1739413095459295 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:15:461.833 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987822147328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:15:482.512 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:15:484.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Router ID configured. Connection to controller initiated 2025-02-13 Thu 02:18:15:488.967 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:489.032 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095488883 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:489.045 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:489.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:489.935 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095488959 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:15:489.945 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:15:489.956 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:489.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095489298 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:489.995 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:490.003 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:490.052 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095489321 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, XMPP:dns-server:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:15:490.065 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:15:490.635 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server [fe80::f816:3eff:fe42:6074%vhost0]:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:15:536.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:536.902 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095536663 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:536.923 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:536.935 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:536.949 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095536905 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 42825 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:536.960 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:542.490 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:42825::10.0.0.17:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-02-13 Thu 02:18:15:542.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-02-13 Thu 02:18:15:543.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:543.633 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095543559 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 42825 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:543.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:544.070 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:544.102 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095543603 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:544.114 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:544.128 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:544.147 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413095543771 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 42317 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:544.155 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:546.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:42317::10.0.0.17:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-02-13 Thu 02:18:15:546.441 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-02-13 Thu 02:18:15:546.656 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:547.051 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413095546618 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 42317 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:548.702 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:548.758 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:548.774 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413095548746 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:548.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:548.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:548.826 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:15:548.879 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413095548801 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:548.977 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:549.234 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:15:549.346 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:15:549.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:549.727 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413095549167 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:549.744 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:549.759 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:549.778 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413095549208 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:549.788 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:549.799 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:549.816 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413095549331 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:549.825 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:551.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:15:551.905 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:552.274 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413095551883 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:552.284 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:553.977 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-02-13 Thu 02:18:15:554.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:555.005 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095554950 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:555.014 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:555.900 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-02-13 Thu 02:18:15:555.998 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:556.017 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095555986 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:556.037 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:556.061 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:556.072 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095556055 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:556.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:556.110 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:556.135 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413095556105 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:15:556.151 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:15:557.460 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:557.484 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413095557443 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:15:557.492 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:15:557.501 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:15:557.526 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413095557495 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:15:557.533 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:16:459.357 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:16:459.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:16:459.430 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:16:459.561 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Idle 2025-02-13 Thu 02:18:16:459.620 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:16:459.681 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:16:459.737 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413096459462 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:16:459.746 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:16:459.756 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:16:459.814 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:16:459.965 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413096459669 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 29, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:16:459.984 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:17:459.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:17:459.951 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:17:460.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:17:460.114 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413097459756 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:17:460.123 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:17:460.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:17:460.295 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413097460062 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 31, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:17:460.317 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:19:460.148 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:19:460.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:19:460.221 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:19:460.680 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 335544539 category: asio.ssl message: short read src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:19:460.710 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Idle 2025-02-13 Thu 02:18:19:460.804 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:19:460.840 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413099460253 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:19:460.851 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:19:460.864 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:19:460.916 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:19:461.085 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413099460793 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 33, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:19:461.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:19:959.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:19:959.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: SANDESH: Send FAILED: 1739413099959539 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:19:959.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987847337728, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:19:962.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:19:962.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413099962298 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:19:962.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:20:296.538 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:20:296.601 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413100296512 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 name = default-domain:default-project:ip-fabric acl_uuid = b1c1623a-323b-4bff-b945-1cec547ea138 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-02-13 Thu 02:18:20:296.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2025-02-13 Thu 02:18:20:301.254 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:20:301.314 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413100301230 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 name = default-domain:default-project:ip-fabric acl_uuid = b1c1623a-323b-4bff-b945-1cec547ea138 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-02-13 Thu 02:18:20:301.325 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2025-02-13 Thu 02:18:20:825.761 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:20:825.817 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413100825735 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 name = default-domain:default-project:ip-fabric acl_uuid = b1c1623a-323b-4bff-b945-1cec547ea138 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-02-13 Thu 02:18:20:825.828 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: VnObjectLog 2025-02-13 Thu 02:18:21:460.859 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:21:461.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:21:461.306 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:21:461.340 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413101460946 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:21:461.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:21:461.357 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:21:461.512 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: SANDESH: Send FAILED: 1739413101461287 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 40, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:21:461.531 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:25:461.347 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:25:461.430 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:25:461.448 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:25:461.606 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Idle 2025-02-13 Thu 02:18:25:461.621 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:25:461.762 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:25:461.815 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413105461510 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:25:461.828 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:25:461.841 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:25:461.891 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:25:462.386 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413105461746 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 42, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:25:462.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:29:461.785 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:29:462.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:29:462.220 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:29:462.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413109461865 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:29:462.277 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:29:462.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:29:462.443 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413109462204 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 44, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:29:462.463 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:34:811.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41300 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:18:34:858.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41300 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:18:37:462.234 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987834742528, Pid 1177849]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:37:462.326 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:37:462.365 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:37:462.477 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Idle 2025-02-13 Thu 02:18:37:462.496 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:37:462.616 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:37:462.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413117462402 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:37:462.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:37:462.664 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:37:462.709 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:37:462.878 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413117462603 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 46, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:37:462.904 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:45:125.727 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:125.831 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413125125673 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric total_acl_rules = 4 interface_list= [ [ (*_iter66) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0, ] ] virtualmachine_list= [ [ ] ] acl = default-domain:default-project:ip-fabric:ip-fabric mirror_acl = ] 2025-02-13 Thu 02:18:45:125.864 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:18:45:126.310 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:126.385 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125126276 [SYS_NOTICE]: UveVMInterfaceAgentTrace: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 ip_address = 10.0.0.17 virtual_network = default-domain:default-project:ip-fabric mac_address = fa:16:3e:42:60:74 label = 17 active = 1 vm_name = l2_active = 1 uuid = 6fd2c88c-dcd0-4807-b0f8-05d81d30aac7 ip6_address = :: ip6_active = 1 floating_ips= [ [ ] ] vm_uuid = ip4_active = 1 is_health_check_active = 1 health_check_instance_list= [ [ ] ] admin_state = 1 fixed_ip4_list= [ [ (*_iter26) = 10.0.0.17, ] ] fixed_ip6_list= [ [ ] ] vn_uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 tx_vlan = 65535 rx_vlan = 65535 parent_interface = ens3 vhostuser_mode = 0 port_mirror_enabled = 0 ] 2025-02-13 Thu 02:18:45:126.398 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVMInterfaceAgentTrace 2025-02-13 Thu 02:18:45:126.408 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:126.422 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125126304 [SYS_NOTICE]: UVEVMITags: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 app = tier = site = deployment = labels= [ [ ] ] custom_tags= [ [ ] ] ] 2025-02-13 Thu 02:18:45:126.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UVEVMITags 2025-02-13 Thu 02:18:45:127.733 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:127.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125127714 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric associated_fip_count = 0 in_bandwidth_usage = 0 out_bandwidth_usage = 0 vrf_stats_list= [ [ [ name = default-domain:default-project:ip-fabric:ip-fabric diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:18:45:127.835 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:18:45:127.844 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:127.909 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125127744 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2025-02-13 Thu 02:18:45:127.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:18:45:127.954 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:128.154 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125127914 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric port_bucket_bmap= [ udp_sport_bitmap= [ [ (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, ] ] udp_dport_bitmap= [ [ (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, ] ] tcp_sport_bitmap= [ [ (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, ] ] tcp_dport_bitmap= [ [ (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, ] ] ] raw_if_stats= [ in_pkts = 48 in_bytes = 2918 out_pkts = 61 out_bytes = 3849 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 0 out_bw_usage = 0 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:18:45:128.172 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:18:45:128.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:128.194 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: SANDESH: Send FAILED: 1739413125127951 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:18:45:128.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987843139328, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:18:45:128.856 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:128.897 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413125128811 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 connected_networks= [ [ (*_iter142) = default-domain:default-project:ip-fabric, ] ] vn_count = 1 ] 2025-02-13 Thu 02:18:45:128.907 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:18:45:128.917 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:128.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413125128869 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 interface_list= [ [ (*_iter143) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0, ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ ] ] vmi_count= [ active = 1 ] down_interface_count = 0 ] 2025-02-13 Thu 02:18:45:128.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:18:45:457.725 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:457.868 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413125457165 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 3 dequeues = 3 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 57 dequeues = 57 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 70 dequeues = 70 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413117462475 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413125128916 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:18:45:457.883 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:45:457.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:458.195 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: SANDESH: Send FAILED: 1739413125457657 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 57, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:45:458.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987838940928, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:45:461.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:462.133 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413125461912 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 xmpp_peer_list= [ [ [ ip = 10.0.0.17 status = 1 setup_time = 1739413095557500 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.17, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2025-02-13 00:26:33.389417","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-7769-1.novalocal","build-id":"unknown","build-number":"unknown"}]} phy_if= [ [ [ name = ens3 mac_address = fa:16:3e:42:60:74 ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:42:60:74 ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_DEBUG log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.17, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.17 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = ens3, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.17 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.17:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.17:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-juju-os-473-1 flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.17:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2025-02-13 00:23:54.879368", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-7769-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2025-02-13 Thu 02:18:45:462.161 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:18:45:462.175 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:462.403 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413125461991 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 48 in_bytes = 2918 out_tpkts = 61 out_bytes = 3849 exception_packets = 22 exception_packets_dropped = 10 exception_packets_allowed = 12 total_flows = 0 aged_flows = 0 uptime = 1739413095126160 phy_if_info= [ [ _iter254->first = ens3 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 48 in_bytes = 2918 out_pkts = 61 out_bytes = 3849 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 35 in_bytes = 2757 out_pkts = 51 out_bytes = 3003 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:18:45:462.424 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:18:45:462.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:45:462.466 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: SANDESH: Send FAILED: 1739413125462067 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 35 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:18:45:462.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987830544128, Pid 1177849]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:18:45:462.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:45:462.825 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:45:463.016 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:45:463.069 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413125462588 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:45:463.083 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:45:463.095 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:45:463.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: SANDESH: Send FAILED: 1739413125462992 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1177849 http_port = 8085 start_time = 1739413095456035 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 62, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:45:463.353 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987817948928, Pid 1177849]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:45:701.698 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53012 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:18:45:751.043 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139987996449088, Pid 1177849]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53012 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:18:51:493.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: vhost interface name : vhost0 2025-02-13 Thu 02:18:51:493.167 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: vhost IP Address : 10.0.0.17/24 2025-02-13 Thu 02:18:51:493.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: vhost gateway : 0.0.0.0 2025-02-13 Thu 02:18:51:493.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Ethernet ports : ens3 2025-02-13 Thu 02:18:51:493.188 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Loopback IP : 0.0.0.0 2025-02-13 Thu 02:18:51:493.194 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Servers : 10.0.0.17:5269 2025-02-13 Thu 02:18:51:493.200 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Authentication : 1 2025-02-13 Thu 02:18:51:493.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:51:493.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:51:493.216 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:51:493.221 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Cluster-Name : 2025-02-13 Thu 02:18:51:493.228 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: DNS Servers : 10.0.0.17:53 2025-02-13 Thu 02:18:51:493.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: DNS client port : 8098 2025-02-13 Thu 02:18:51:493.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: DNS timeout : 3000 2025-02-13 Thu 02:18:51:493.269 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: DNS max retries : 2 2025-02-13 Thu 02:18:51:493.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Dns Authentication : 1 2025-02-13 Thu 02:18:51:493.287 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Server Certificate : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:51:493.292 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp Server Key : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:51:493.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Xmpp CA Certificate : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:51:493.303 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TSN Servers : [] 2025-02-13 Thu 02:18:51:493.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: COLLECTOR Servers : 10.0.0.17:8086 2025-02-13 Thu 02:18:51:493.313 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Tunnel-Type : MPLSoGRE 2025-02-13 Thu 02:18:51:493.318 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Metadata-Proxy Shared Secret: c9392c64-153a-4aa0-b00d-4fa0ced2fcc1 2025-02-13 Thu 02:18:51:493.323 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Metadata-Proxy Port : 8097 2025-02-13 Thu 02:18:51:493.328 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Metadata-Proxy SSL Flag : 0 2025-02-13 Thu 02:18:51:493.341 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Max Vm Flows : 100 2025-02-13 Thu 02:18:51:493.346 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Linklocal Max System Flows : 16384 2025-02-13 Thu 02:18:51:493.351 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Linklocal Max Vm Flows : 16384 2025-02-13 Thu 02:18:51:493.356 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow cache timeout : 0 2025-02-13 Thu 02:18:51:493.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Stale Interface cleanup timeout : 60 2025-02-13 Thu 02:18:51:493.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow thread count : 1 2025-02-13 Thu 02:18:51:493.371 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow latency limit : 0 2025-02-13 Thu 02:18:51:493.376 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow index-mgr sm log count : 0 2025-02-13 Thu 02:18:51:493.381 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow add-tokens : 50 2025-02-13 Thu 02:18:51:493.386 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow ksync-tokens : 25 2025-02-13 Thu 02:18:51:493.390 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow del-tokens : 16 2025-02-13 Thu 02:18:51:493.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow update-tokens : 16 2025-02-13 Thu 02:18:51:493.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Pin flow netlink task to CPU: 2025-02-13 Thu 02:18:51:493.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Maximum sessions : 100 2025-02-13 Thu 02:18:51:493.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Maximum session aggregates : 8 2025-02-13 Thu 02:18:51:493.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Maximum session endpoints : 5 2025-02-13 Thu 02:18:51:493.419 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Fabric SNAT hash table size : 4096 2025-02-13 Thu 02:18:51:493.424 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Flow excluding Router ID in hash :0 2025-02-13 Thu 02:18:51:493.432 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Agent Mode : Vrouter 2025-02-13 Thu 02:18:51:493.437 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Gateway Mode : None 2025-02-13 Thu 02:18:51:493.442 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: DHCP Relay Mode : 0 2025-02-13 Thu 02:18:51:493.447 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance netns cmd : /usr/local/bin/opencontrail-vrouter-netns 2025-02-13 Thu 02:18:51:493.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance docker cmd : /usr/local/bin/opencontrail-vrouter-docker 2025-02-13 Thu 02:18:51:493.457 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance workers : 0 2025-02-13 Thu 02:18:51:493.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance timeout : 0 2025-02-13 Thu 02:18:51:493.466 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance lb ssl : 2025-02-13 Thu 02:18:51:493.471 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Service instance lbaas auth : 2025-02-13 Thu 02:18:51:493.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Bgp as a service port range : 50000-50512 2025-02-13 Thu 02:18:51:493.481 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Services queue limit : 1024 2025-02-13 Thu 02:18:51:493.486 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: BGPAAS max shared sessions for service port : 4 2025-02-13 Thu 02:18:51:493.491 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Sandesh Key file : /etc/contrail/ssl/private/server-privkey.pem 2025-02-13 Thu 02:18:51:493.496 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Sandesh Cert file : /etc/contrail/ssl/certs/server.pem 2025-02-13 Thu 02:18:51:493.500 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Sandesh CA Cert : /etc/contrail/ssl/certs/ca-cert.pem 2025-02-13 Thu 02:18:51:493.505 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Sandesh SSL Enable : 1 2025-02-13 Thu 02:18:51:493.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Introspect SSL Enable : 1 2025-02-13 Thu 02:18:51:493.515 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Hypervisor mode : kvm 2025-02-13 Thu 02:18:51:493.615 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Ethernet Port Encap Type : 2025-02-13 Thu 02:18:51:493.624 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: Platform mode : Vrouter on host linux kernel 2025-02-13 Thu 02:18:51:493.815 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: VRouter [SYS_INFO]: VersionInfoLog: {"build-info":[{"build-version":"nightly","build-time":"2025-02-13 00:26:33.389417","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-7769-1.novalocal","build-id":"unknown","build-number":"unknown"}]} src/contrail-common/base/misc_utils.cc 41 2025-02-13 Thu 02:18:51:493.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Logging: DISABLED -> ENABLED 2025-02-13 Thu 02:18:51:493.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Logging: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] log4level: [ NOTSET ] -> [ DEBUG ] 2025-02-13 Thu 02:18:51:500.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.273 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> self 2025-02-13 Thu 02:18:51:500.281 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:51:500.287 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:51:500.293 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-access-control-list -> self 2025-02-13 Thu 02:18:51:500.304 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap access-control-list : self -> virtual-network-access-control-list 2025-02-13 Thu 02:18:51:500.310 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap access-control-list : virtual-network-access-control-list -> nil 2025-02-13 Thu 02:18:51:500.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> self 2025-02-13 Thu 02:18:51:500.327 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:51:500.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> virtual-network-network-ipam 2025-02-13 Thu 02:18:51:500.338 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:51:500.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-qos-config -> self 2025-02-13 Thu 02:18:51:500.356 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-config : self -> virtual-network-qos-config 2025-02-13 Thu 02:18:51:500.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-config : virtual-network-qos-config -> nil 2025-02-13 Thu 02:18:51:500.367 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.388 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-security-logging-object -> self 2025-02-13 Thu 02:18:51:500.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : self -> virtual-network-security-logging-object 2025-02-13 Thu 02:18:51:500.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : virtual-network-security-logging-object -> nil 2025-02-13 Thu 02:18:51:500.406 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.417 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : logical-router-virtual-network -> self 2025-02-13 Thu 02:18:51:500.423 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router-virtual-network : self -> logical-router-virtual-network 2025-02-13 Thu 02:18:51:500.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router-virtual-network : logical-router-virtual-network -> logical-router-virtual-network 2025-02-13 Thu 02:18:51:500.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router : self -> logical-router-virtual-network 2025-02-13 Thu 02:18:51:500.440 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router : logical-router-virtual-network -> nil 2025-02-13 Thu 02:18:51:500.446 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-network 2025-02-13 Thu 02:18:51:500.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-multicast-policy -> self 2025-02-13 Thu 02:18:51:500.457 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap multicast-policy : self -> virtual-network-multicast-policy 2025-02-13 Thu 02:18:51:500.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap multicast-policy : virtual-network-multicast-policy -> nil 2025-02-13 Thu 02:18:51:500.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for routing-instance 2025-02-13 Thu 02:18:51:500.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> self 2025-02-13 Thu 02:18:51:500.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:51:500.484 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-routing-instance 2025-02-13 Thu 02:18:51:500.489 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:51:500.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:51:500.502 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for security-group 2025-02-13 Thu 02:18:51:500.507 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-group : security-group-access-control-list -> self 2025-02-13 Thu 02:18:51:500.512 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap access-control-list : self -> security-group-access-control-list 2025-02-13 Thu 02:18:51:500.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap access-control-list : security-group-access-control-list -> nil 2025-02-13 Thu 02:18:51:500.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for tag 2025-02-13 Thu 02:18:51:500.529 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> self 2025-02-13 Thu 02:18:51:500.538 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:500.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : policy-management-application-policy-set -> application-policy-set-tag 2025-02-13 Thu 02:18:51:500.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap policy-management : policy-management-application-policy-set -> nil 2025-02-13 Thu 02:18:51:500.555 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.560 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:51:500.566 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:500.571 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:500.577 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:51:500.582 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:51:500.588 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.593 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-02-13 Thu 02:18:51:500.598 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:51:500.604 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : virtual-machine-interface-virtual-machine -> nil 2025-02-13 Thu 02:18:51:500.609 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.614 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-02-13 Thu 02:18:51:500.620 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:51:500.625 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> nil 2025-02-13 Thu 02:18:51:500.631 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.636 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:51:500.641 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:500.649 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:500.654 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:51:500.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:51:500.666 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.671 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-routing-instance -> self 2025-02-13 Thu 02:18:51:500.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-routing-instance : self -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:51:500.682 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-routing-instance : virtual-machine-interface-routing-instance -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:51:500.688 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : self -> virtual-machine-interface-routing-instance 2025-02-13 Thu 02:18:51:500.693 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : virtual-machine-interface-routing-instance -> nil 2025-02-13 Thu 02:18:51:500.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.705 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-group -> self 2025-02-13 Thu 02:18:51:500.710 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-group : self -> virtual-machine-interface-security-group 2025-02-13 Thu 02:18:51:500.715 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-group : virtual-machine-interface-security-group -> nil 2025-02-13 Thu 02:18:51:500.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.726 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : alias-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:500.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap alias-ip : self -> alias-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.738 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap alias-ip : alias-ip-pool-alias-ip -> alias-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.744 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap alias-ip-pool : virtual-network-alias-ip-pool -> alias-ip-pool-alias-ip 2025-02-13 Thu 02:18:51:500.749 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-alias-ip-pool 2025-02-13 Thu 02:18:51:500.755 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-alias-ip-pool -> nil 2025-02-13 Thu 02:18:51:500.765 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.770 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:500.776 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.787 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-02-13 Thu 02:18:51:500.793 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:51:500.798 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:51:500.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-provider-network 2025-02-13 Thu 02:18:51:500.808 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-provider-network -> nil 2025-02-13 Thu 02:18:51:500.814 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:500.825 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.830 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : floating-ip-pool-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.871 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip-pool : virtual-network-floating-ip-pool -> floating-ip-pool-floating-ip 2025-02-13 Thu 02:18:51:500.880 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:51:500.887 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-floating-ip-pool 2025-02-13 Thu 02:18:51:500.892 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:51:500.898 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:51:500.920 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.926 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : floating-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:500.931 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : self -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap floating-ip : instance-ip-floating-ip -> floating-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.946 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap instance-ip : instance-ip-virtual-network -> instance-ip-floating-ip 2025-02-13 Thu 02:18:51:500.952 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> instance-ip-virtual-network 2025-02-13 Thu 02:18:51:500.957 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : instance-ip-virtual-network -> nil 2025-02-13 Thu 02:18:51:500.963 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : instance-ip-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:500.974 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap instance-ip : self -> instance-ip-virtual-machine-interface 2025-02-13 Thu 02:18:51:500.979 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap instance-ip : instance-ip-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:51:500.985 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:500.995 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-route-table -> self 2025-02-13 Thu 02:18:51:501.002 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap interface-route-table : self -> virtual-machine-interface-route-table 2025-02-13 Thu 02:18:51:501.007 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap interface-route-table : virtual-machine-interface-route-table -> nil 2025-02-13 Thu 02:18:51:501.013 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.018 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : subnet-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap subnet : self -> subnet-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.040 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap subnet : subnet-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:51:501.047 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.052 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.087 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> logical-interface-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.096 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-02-13 Thu 02:18:51:501.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:51:501.115 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.130 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-port-group-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.136 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> virtual-port-group-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.141 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : physical-router-physical-interface -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:51:501.147 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:51:501.152 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.159 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.170 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-as-a-service : bgpaas-bgp-router -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-router : self -> bgpaas-bgp-router 2025-02-13 Thu 02:18:51:501.187 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-router : instance-bgp-router -> bgpaas-bgp-router 2025-02-13 Thu 02:18:51:501.192 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : self -> instance-bgp-router 2025-02-13 Thu 02:18:51:501.198 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : instance-bgp-router -> nil 2025-02-13 Thu 02:18:51:501.204 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.209 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : bgpaas-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.214 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-as-a-service : self -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.219 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bgp-as-a-service : bgpaas-health-check -> bgpaas-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.226 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-health-check : self -> bgpaas-health-check 2025-02-13 Thu 02:18:51:501.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-health-check : bgpaas-health-check -> nil 2025-02-13 Thu 02:18:51:501.239 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.244 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-qos-config -> self 2025-02-13 Thu 02:18:51:501.250 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-config : self -> virtual-machine-interface-qos-config 2025-02-13 Thu 02:18:51:501.255 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-config : virtual-machine-interface-qos-config -> nil 2025-02-13 Thu 02:18:51:501.261 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.266 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-security-logging-object -> self 2025-02-13 Thu 02:18:51:501.271 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : self -> virtual-machine-interface-security-logging-object 2025-02-13 Thu 02:18:51:501.277 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : virtual-machine-interface-security-logging-object -> nil 2025-02-13 Thu 02:18:51:501.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-02-13 Thu 02:18:51:501.294 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.299 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.306 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.311 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:51:501.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-02-13 Thu 02:18:51:501.327 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-bridge-domain -> self 2025-02-13 Thu 02:18:51:501.337 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.345 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface-bridge-domain : virtual-machine-interface-bridge-domain -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.350 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : self -> virtual-machine-interface-bridge-domain 2025-02-13 Thu 02:18:51:501.355 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : virtual-machine-interface-bridge-domain -> nil 2025-02-13 Thu 02:18:51:501.362 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.367 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-tag -> self 2025-02-13 Thu 02:18:51:501.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> virtual-machine-interface-tag 2025-02-13 Thu 02:18:51:501.377 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-interface-tag 2025-02-13 Thu 02:18:51:501.382 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.388 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.394 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.399 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:501.417 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.422 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> self 2025-02-13 Thu 02:18:51:501.428 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:501.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-tag -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:501.439 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> virtual-network-tag 2025-02-13 Thu 02:18:51:501.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> virtual-network-tag 2025-02-13 Thu 02:18:51:501.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.457 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.467 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.472 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.477 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:501.483 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.488 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> self 2025-02-13 Thu 02:18:51:501.493 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:51:501.499 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:51:501.504 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-02-13 Thu 02:18:51:501.509 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-02-13 Thu 02:18:51:501.514 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.519 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.529 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:501.546 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.551 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-sub-interface -> self 2025-02-13 Thu 02:18:51:501.556 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:51:501.566 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-machine -> virtual-machine-interface-sub-interface 2025-02-13 Thu 02:18:51:501.571 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : self -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:51:501.576 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine : virtual-machine-tag -> virtual-machine-interface-virtual-machine 2025-02-13 Thu 02:18:51:501.581 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> virtual-machine-tag 2025-02-13 Thu 02:18:51:501.586 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> virtual-machine-tag 2025-02-13 Thu 02:18:51:501.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.601 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.606 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.611 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.616 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:501.623 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : project-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.634 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap project : self -> project-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.639 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap project : project-tag -> project-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.650 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> project-tag 2025-02-13 Thu 02:18:51:501.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : application-policy-set-tag -> project-tag 2025-02-13 Thu 02:18:51:501.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : self -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.665 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> application-policy-set-tag 2025-02-13 Thu 02:18:51:501.670 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.678 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.683 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:501.688 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:501.694 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> self 2025-02-13 Thu 02:18:51:501.705 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.710 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> nil 2025-02-13 Thu 02:18:51:501.727 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for virtual-machine-interface 2025-02-13 Thu 02:18:51:501.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : logical-router-interface -> self 2025-02-13 Thu 02:18:51:501.737 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router : self -> logical-router-interface 2025-02-13 Thu 02:18:51:501.743 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-router : logical-router-interface -> nil 2025-02-13 Thu 02:18:51:501.749 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for physical-interface 2025-02-13 Thu 02:18:51:501.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : physical-router-physical-interface -> self 2025-02-13 Thu 02:18:51:501.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:51:501.764 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.770 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for physical-interface 2025-02-13 Thu 02:18:51:501.775 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : virtual-port-group-physical-interface -> self 2025-02-13 Thu 02:18:51:501.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-port-group-physical-interface : self -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:51:501.787 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-port-group-physical-interface : virtual-port-group-physical-interface -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:51:501.794 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-port-group : self -> virtual-port-group-physical-interface 2025-02-13 Thu 02:18:51:501.800 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-port-group : virtual-port-group-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.806 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:51:501.811 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-interface : physical-interface-logical-interface -> self 2025-02-13 Thu 02:18:51:501.816 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : self -> physical-interface-logical-interface 2025-02-13 Thu 02:18:51:501.821 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : physical-router-physical-interface -> physical-interface-logical-interface 2025-02-13 Thu 02:18:51:501.826 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:51:501.831 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.836 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:51:501.841 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-interface : logical-interface-virtual-machine-interface -> self 2025-02-13 Thu 02:18:51:501.847 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> logical-interface-virtual-machine-interface 2025-02-13 Thu 02:18:51:501.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : logical-interface-virtual-machine-interface -> nil 2025-02-13 Thu 02:18:51:501.858 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for logical-interface 2025-02-13 Thu 02:18:51:501.876 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap logical-interface : physical-router-logical-interface -> self 2025-02-13 Thu 02:18:51:501.882 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : self -> physical-router-logical-interface 2025-02-13 Thu 02:18:51:501.887 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-logical-interface -> nil 2025-02-13 Thu 02:18:51:501.893 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for physical-router 2025-02-13 Thu 02:18:51:501.898 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-router : physical-router-physical-interface -> self 2025-02-13 Thu 02:18:51:501.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : self -> physical-router-physical-interface 2025-02-13 Thu 02:18:51:501.909 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap physical-interface : physical-router-physical-interface -> nil 2025-02-13 Thu 02:18:51:501.918 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for service-health-check 2025-02-13 Thu 02:18:51:501.923 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-02-13 Thu 02:18:51:501.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-02-13 Thu 02:18:51:501.935 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> service-port-health-check 2025-02-13 Thu 02:18:51:501.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:501.945 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:51:501.951 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:51:501.958 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for service-health-check 2025-02-13 Thu 02:18:51:501.963 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-health-check : service-port-health-check -> self 2025-02-13 Thu 02:18:51:501.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> service-port-health-check 2025-02-13 Thu 02:18:51:501.973 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : port-tuple-interface -> service-port-health-check 2025-02-13 Thu 02:18:51:501.978 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap port-tuple : self -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.983 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap port-tuple : port-tuple-interface -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.989 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : self -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.994 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-machine-interface : virtual-machine-interface-virtual-network -> port-tuple-interface 2025-02-13 Thu 02:18:51:501.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-network-ipam -> virtual-machine-interface-virtual-network 2025-02-13 Thu 02:18:51:502.004 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : self -> virtual-network-network-ipam 2025-02-13 Thu 02:18:51:502.034 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network-network-ipam : virtual-network-network-ipam -> nil 2025-02-13 Thu 02:18:51:502.042 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for qos-config 2025-02-13 Thu 02:18:51:502.047 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-config : global-qos-config-qos-config -> self 2025-02-13 Thu 02:18:51:502.054 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap global-qos-config : global-qos-config-qos-config -> nil 2025-02-13 Thu 02:18:51:502.065 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for forwarding-class 2025-02-13 Thu 02:18:51:502.070 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap forwarding-class : forwarding-class-qos-queue -> self 2025-02-13 Thu 02:18:51:502.077 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-queue : self -> forwarding-class-qos-queue 2025-02-13 Thu 02:18:51:502.082 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap qos-queue : forwarding-class-qos-queue -> nil 2025-02-13 Thu 02:18:51:502.088 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:51:502.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : security-logging-object-network-policy -> self 2025-02-13 Thu 02:18:51:502.104 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap network-policy : self -> security-logging-object-network-policy 2025-02-13 Thu 02:18:51:502.109 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap network-policy : security-logging-object-network-policy -> nil 2025-02-13 Thu 02:18:51:502.115 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:51:502.120 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : security-logging-object-security-group -> self 2025-02-13 Thu 02:18:51:502.126 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-group : self -> security-logging-object-security-group 2025-02-13 Thu 02:18:51:502.131 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-group : security-logging-object-security-group -> nil 2025-02-13 Thu 02:18:51:502.137 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:51:502.142 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : firewall-policy-security-logging-object -> self 2025-02-13 Thu 02:18:51:502.148 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-security-logging-object : self -> firewall-policy-security-logging-object 2025-02-13 Thu 02:18:51:502.153 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-security-logging-object : firewall-policy-security-logging-object -> firewall-policy-security-logging-object 2025-02-13 Thu 02:18:51:502.159 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : firewall-policy-security-logging-object -> nil 2025-02-13 Thu 02:18:51:502.165 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for security-logging-object 2025-02-13 Thu 02:18:51:502.170 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap security-logging-object : firewall-rule-security-logging-object -> self 2025-02-13 Thu 02:18:51:502.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule-security-logging-object : self -> firewall-rule-security-logging-object 2025-02-13 Thu 02:18:51:502.181 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule-security-logging-object : firewall-rule-security-logging-object -> firewall-rule-security-logging-object 2025-02-13 Thu 02:18:51:502.190 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : firewall-rule-security-logging-object -> nil 2025-02-13 Thu 02:18:51:502.203 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for bridge-domain 2025-02-13 Thu 02:18:51:502.208 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-02-13 Thu 02:18:51:502.214 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:51:502.219 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-routing-instance -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:51:502.225 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : self -> virtual-network-routing-instance 2025-02-13 Thu 02:18:51:502.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap routing-instance : virtual-network-routing-instance -> nil 2025-02-13 Thu 02:18:51:502.235 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for bridge-domain 2025-02-13 Thu 02:18:51:502.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap bridge-domain : virtual-network-bridge-domain -> self 2025-02-13 Thu 02:18:51:502.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : self -> virtual-network-bridge-domain 2025-02-13 Thu 02:18:51:502.251 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap virtual-network : virtual-network-bridge-domain -> nil 2025-02-13 Thu 02:18:51:502.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for application-policy-set 2025-02-13 Thu 02:18:51:502.262 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set : application-policy-set-firewall-policy -> self 2025-02-13 Thu 02:18:51:502.268 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:502.273 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap application-policy-set-firewall-policy : application-policy-set-firewall-policy -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:502.278 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : self -> application-policy-set-firewall-policy 2025-02-13 Thu 02:18:51:502.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : application-policy-set-firewall-policy -> nil 2025-02-13 Thu 02:18:51:502.290 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:51:502.295 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:51:502.301 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.314 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.319 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : firewall-rule-service-group -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.325 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-group : self -> firewall-rule-service-group 2025-02-13 Thu 02:18:51:502.331 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap service-group : firewall-rule-service-group -> nil 2025-02-13 Thu 02:18:51:502.337 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:51:502.342 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:51:502.347 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.352 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.357 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.362 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : firewall-rule-address-group -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.368 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap address-group : self -> firewall-rule-address-group 2025-02-13 Thu 02:18:51:502.373 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap address-group : address-group-tag -> firewall-rule-address-group 2025-02-13 Thu 02:18:51:502.379 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> address-group-tag 2025-02-13 Thu 02:18:51:502.384 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : address-group-tag -> nil 2025-02-13 Thu 02:18:51:502.390 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:51:502.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:51:502.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : firewall-rule-tag -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.424 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : self -> firewall-rule-tag 2025-02-13 Thu 02:18:51:502.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap tag : firewall-rule-tag -> nil 2025-02-13 Thu 02:18:51:502.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Updating dependency tacker rules for firewall-policy 2025-02-13 Thu 02:18:51:502.440 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy : firewall-policy-firewall-rule -> self 2025-02-13 Thu 02:18:51:502.445 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.450 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-policy-firewall-rule : firewall-policy-firewall-rule -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.455 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : self -> firewall-policy-firewall-rule 2025-02-13 Thu 02:18:51:502.460 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Adding ReactorMap firewall-rule : firewall-policy-firewall-rule -> nil 2025-02-13 Thu 02:18:51:641.212 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 10.0.0.17:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:51:641.427 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Current receive sock buffer size is 262144 2025-02-13 Thu 02:18:51:641.505 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Vrouter family is 34 2025-02-13 Thu 02:18:51:641.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Flow table size : 20185088 2025-02-13 Thu 02:18:51:641.734 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672711993088, Pid 1180635]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2025-02-13 Thu 02:18:51:656.119 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Flow table size : 161218560 2025-02-13 Thu 02:18:51:656.165 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SetHugePages: Bridge table size:20185088 Flow table size:161218560 2025-02-13 Thu 02:18:51:656.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Mem mapping hugepage file:/dev/hugepages2M/bridge size:20185088 2025-02-13 Thu 02:18:51:656.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Mem mapped hugepage file:/dev/hugepages2M/bridge to addr:0x7f0813200000 2025-02-13 Thu 02:18:51:656.232 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Mem mapping hugepage file:/dev/hugepages2M/flow size:161218560 2025-02-13 Thu 02:18:51:656.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Mem mapped hugepage file:/dev/hugepages2M/flow to addr:0x7f0809800000 2025-02-13 Thu 02:18:51:656.253 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Sending Huge Page configuration to VROUTER 2025-02-13 Thu 02:18:51:656.303 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Vrouter using already allocated memory 2025-02-13 Thu 02:18:51:670.507 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Flow table size : 161218560 2025-02-13 Thu 02:18:51:670.678 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Flow table size : 20185088 2025-02-13 Thu 02:18:51:728.423 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: ROLE : Generator 2025-02-13 Thu 02:18:51:728.464 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: MODULE : contrail-vrouter-agent 2025-02-13 Thu 02:18:51:728.471 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: SOURCE : cn-jenkins-deploy-platform-juju-os-473-1 2025-02-13 Thu 02:18:51:728.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: NODE TYPE : Compute 2025-02-13 Thu 02:18:51:728.481 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: INSTANCE ID : 0 2025-02-13 Thu 02:18:51:728.486 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: HTTP SERVER PORT : 8085 2025-02-13 Thu 02:18:51:728.492 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: System Log Send Rate Limit: 0 -> 0 2025-02-13 Thu 02:18:51:728.834 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: HTTP Introspect Init 2025-02-13 Thu 02:18:51:729.061 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:8085 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:51:729.122 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Sandesh Http Server Port: 8085 2025-02-13 Thu 02:18:51:729.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: NOT Writing http_port 8085TO : /tmp/contrail-vrouter-agent.1180204.http_port 2025-02-13 Thu 02:18:51:729.191 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: CONNECT TO COLLECTOR: 1 2025-02-13 Thu 02:18:51:729.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Idle 2025-02-13 Thu 02:18:51:729.456 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: No Client: 1739413131729377 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:51:729.756 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvStart in state Idle 2025-02-13 Thu 02:18:51:729.877 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Disconnect 2025-02-13 Thu 02:18:51:729.977 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvCollectorUpdate in state Disconnect 2025-02-13 Thu 02:18:51:730.317 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:51:730.397 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.430 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131729810 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-02-13 Thu 02:18:51:730.438 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-02-13 Thu 02:18:51:730.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.464 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131729849 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2025-02-13 Thu 02:18:51:730.470 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrfObjectLog 2025-02-13 Thu 02:18:51:730.477 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.502 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131729866 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:51:730.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:730.518 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.592 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131729967 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:51:730.609 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:51:730.619 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.693 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface lo flags 65609 2025-02-13 Thu 02:18:51:730.703 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131730048 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:51:730.738 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:51:730.720 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface ens3 flags 69699 2025-02-13 Thu 02:18:51:730.752 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.784 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131730068 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:51:730.790 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:730.793 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface fan-252 flags 69699 2025-02-13 Thu 02:18:51:730.813 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface ftun0 flags 69699 2025-02-13 Thu 02:18:51:730.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:730.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface lxdbr0 flags 4099 2025-02-13 Thu 02:18:51:730.861 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vethcbdd4c17 flags 69699 2025-02-13 Thu 02:18:51:730.891 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131730387 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:51:730.910 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:51:730.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth48710ba3 flags 69699 2025-02-13 Thu 02:18:51:730.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth42eea4cc flags 69699 2025-02-13 Thu 02:18:51:730.962 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vethfc72e571 flags 69699 2025-02-13 Thu 02:18:51:730.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth74295baa flags 69699 2025-02-13 Thu 02:18:51:730.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth0c8bdc3e flags 69699 2025-02-13 Thu 02:18:51:731.005 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vetheea8ac85 flags 69699 2025-02-13 Thu 02:18:51:731.056 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vethfa20855f flags 69699 2025-02-13 Thu 02:18:51:731.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth772e951b flags 69699 2025-02-13 Thu 02:18:51:731.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vethc874d1a9 flags 69699 2025-02-13 Thu 02:18:51:731.099 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface veth5396329c flags 69699 2025-02-13 Thu 02:18:51:731.127 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vethe535c679 flags 69699 2025-02-13 Thu 02:18:51:731.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface docker0 flags 4099 2025-02-13 Thu 02:18:51:731.157 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface pkt1 flags 65601 2025-02-13 Thu 02:18:51:731.162 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface pkt3 flags 65601 2025-02-13 Thu 02:18:51:731.168 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface pkt2 flags 65601 2025-02-13 Thu 02:18:51:731.179 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface vhost0 flags 69699 2025-02-13 Thu 02:18:51:731.185 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink interface message RTM_NEWLINK for interface pkt0 flags 69699 2025-02-13 Thu 02:18:51:731.228 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface lo 2025-02-13 Thu 02:18:51:731.238 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-02-13 Thu 02:18:51:731.248 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface lxdbr0 2025-02-13 Thu 02:18:51:731.259 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface docker0 2025-02-13 Thu 02:18:51:731.268 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-02-13 Thu 02:18:51:731.286 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface lo 2025-02-13 Thu 02:18:51:731.297 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface fan-252 2025-02-13 Thu 02:18:51:731.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface ftun0 2025-02-13 Thu 02:18:51:731.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface vhost0 2025-02-13 Thu 02:18:51:731.335 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink address message RTM_NEWADDR for interface pkt0 2025-02-13 Thu 02:18:51:731.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.0.0.0/24 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.455 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.0.0.1/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.467 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.0.0.2/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.477 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.0.0.4/32 Interface vhost0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.488 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 10.200.97.0/24 Interface lxdbr0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.493 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.504 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 172.17.0.0/16 Interface docker0 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.514 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Handle netlink route message RTM_NEWROUTE : 252.0.0.0/8 Interface fan-252 GW 0.0.0.0 2025-02-13 Thu 02:18:51:731.519 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:51:731.529 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:51:731.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:51:731.554 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.559 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:51:731.563 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 2 scope 254 2025-02-13 Thu 02:18:51:731.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 2 table 255 type 3 scope 253 2025-02-13 Thu 02:18:51:731.585 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.594 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.599 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.604 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.609 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 254 type 1 scope 0 2025-02-13 Thu 02:18:51:731.614 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:51:731.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:51:731.623 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:51:731.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 4 scope 0 2025-02-13 Thu 02:18:51:731.633 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:51:731.638 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:51:731.643 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 2 scope 0 2025-02-13 Thu 02:18:51:731.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:51:731.652 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:51:731.657 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:51:731.662 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:51:731.667 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Ignoring Netlink route with family 10 table 255 type 5 scope 0 2025-02-13 Thu 02:18:51:750.506 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server 0.0.0.0:9091 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:51:750.766 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Router ID configured. Connection to controller initiated 2025-02-13 Thu 02:18:51:751.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:751.379 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413131751245 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:751.390 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:751.411 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:751.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413131751318 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2025-02-13 Thu 02:18:51:751.471 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:751.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:751.635 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413131751574 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:751.642 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:751.651 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:751.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413131751608 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.17, XMPP:dns-server:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:51:751.688 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:752.538 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: TCP [SYS_DEBUG]: TcpServerMessageLog: Server [fe80::f816:3eff:fe42:6074%vhost0]:8097 Initialization complete src/contrail-common/io/tcp_server.cc 137 2025-02-13 Thu 02:18:51:800.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:800.913 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413131800808 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:800.925 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:801.053 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:801.112 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131801005 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 39505 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:801.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:805.419 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:39505::10.0.0.17:5269 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-02-13 Thu 02:18:51:805.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-02-13 Thu 02:18:51:805.659 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:805.686 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131805641 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 39505 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:805.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:805.720 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:805.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131805712 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:805.738 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:806.264 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:806.305 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413131806227 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect event = Connect Event local_port = 37069 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:806.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:810.007 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:37069::10.0.0.17:8093 < Active session connection complete src/contrail-common/io/tcp_session.cc 282 2025-02-13 Thu 02:18:51:810.055 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.17 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2025-02-13 Thu 02:18:51:810.169 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:810.189 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131810158 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 37069 remote_port = 8093 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:810.197 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:810.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:810.350 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131810277 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:810.364 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:810.377 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:810.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413131810294 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:810.404 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:810.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:51:811.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:51:811.604 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:51:811.616 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:812.049 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413131811513 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenSent next_state = Open Confirm event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:812.061 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:812.077 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:812.096 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413131811542 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Sent Start Tls, OpenConfirm Feature Negotiation event = EvStreamFeatureRequest ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:812.103 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:812.110 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:812.120 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413131811595 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-control@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:812.126 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:812.278 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: XMPP [SYS_INFO]: XmppUnconsumedEvent: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1/dns:10.0.0.17 RECV Mode Client: Unconsumed Event xmsm::EvXmppKeepalive in state OpenConfirm controller/src/xmpp/xmpp_state_machine.cc 1649 2025-02-13 Thu 02:18:51:812.287 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:812.369 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413131812261 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Trigger Client Ssl Handshake event = EvTlsProceed identifier = network-dns@contrailsystems.com ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:812.383 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:816.272 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-02-13 Thu 02:18:51:816.393 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:816.413 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413131816380 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:816.421 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:818.530 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: XMPP [SYS_DEBUG]: XmppSslHandShakeMessage: XMPP Peer cn-jenkins-deploy-platform-juju-os-473-1:10.0.0.17 RECV success controller/src/xmpp/xmpp_connection.cc 402 2025-02-13 Thu 02:18:51:818.577 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:818.621 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131818561 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:818.634 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:818.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:818.659 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131818609 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = OpenConfirm Feature Negotiation Success event = EvTlsHandShakeSuccess ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:818.665 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:818.674 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:818.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413131818648 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:dns-server:10.0.0.17, Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:51:818.706 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:822.436 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:822.467 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413131822417 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.17 state = OpenConfirm next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-juju-os-473-1 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2025-02-13 Thu 02:18:51:822.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: XmppConnectionLog 2025-02-13 Thu 02:18:51:822.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:822.519 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413131822487 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self ], ] ] ] 2025-02-13 Thu 02:18:51:822.535 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:870.570 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:870.683 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413131870539 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:51:870.698 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:918.725 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:918.790 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413131918697 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:51:918.801 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:51:919.336 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:919.426 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413131919296 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Addition uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 name = default-domain:default-project:ip-fabric acl_uuid = b1c1623a-323b-4bff-b945-1cec547ea138 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-02-13 Thu 02:18:51:919.438 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-02-13 Thu 02:18:51:925.147 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:51:925.207 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413131925125 AgentVn [SYS_INFO]: VnObjectLog: vn= [ event = Modification uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 name = default-domain:default-project:ip-fabric acl_uuid = b1c1623a-323b-4bff-b945-1cec547ea138 vrf = default-domain:default-project:ip-fabric:ip-fabric ipv4_forwarding = 1 bridging = 1 admin_state = 1 ] file = controller/src/vnsw/agent/oper/vn.cc line = 1391 2025-02-13 Thu 02:18:51:925.227 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VnObjectLog 2025-02-13 Thu 02:18:52:730.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:52:730.535 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:52:730.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:52:730.746 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Idle 2025-02-13 Thu 02:18:52:730.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:52:730.863 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:52:730.894 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413132730613 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:52:730.902 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:52:730.911 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:52:730.970 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:52:731.160 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413132730841 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 33, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:52:731.187 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:53:730.846 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:53:731.138 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:53:731.241 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:53:731.271 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413133730927 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:53:731.290 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:53:731.298 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:53:731.460 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413133731226 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 35, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:53:731.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:55:731.277 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:55:731.351 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:18:55:731.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:18:55:731.563 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:18:55:731.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:18:55:731.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:55:731.730 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413135731465 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:55:731.738 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:55:731.747 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:18:55:731.790 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:18:55:731.971 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413135731682 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 37, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:55:732.000 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:18:57:731.672 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:18:57:732.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:18:57:732.169 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:57:732.214 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413137731764 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:18:57:732.227 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:18:57:732.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:18:57:732.428 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413137732152 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 15, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 39, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:18:57:732.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:01:732.178 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:01:732.299 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:01:732.334 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:19:01:732.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Idle 2025-02-13 Thu 02:19:01:732.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:19:01:732.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:01:732.631 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413141732383 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:01:732.642 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:01:732.654 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:19:01:732.713 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:01:732.934 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413141732585 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 16, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 41, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:01:732.963 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:04:747.135 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33510 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:19:04:790.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33510 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:19:05:732.619 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:19:05:733.003 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:19:05:733.121 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:05:733.163 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413145732751 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:05:733.175 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:05:733.188 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:05:733.374 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413145733104 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 43, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:05:733.406 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:13:733.151 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:13:733.244 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:13:733.527 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:19:13:733.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Idle 2025-02-13 Thu 02:19:13:733.802 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 335544539 category: asio.ssl message: short read src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:19:13:733.887 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:13:733.918 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413153733563 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:13:733.926 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:13:733.935 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:19:13:733.973 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:13:734.227 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413153733873 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 45, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:13:734.256 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:14:954.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:58376 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:19:14:998.947 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:58376 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:19:21:494.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:494.423 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161494310 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric total_acl_rules = 4 interface_list= [ [ (*_iter66) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0, ] ] virtualmachine_list= [ [ ] ] acl = default-domain:default-project:ip-fabric:ip-fabric mirror_acl = ] 2025-02-13 Thu 02:19:21:494.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:19:21:494.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:494.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161494394 [SYS_NOTICE]: UveVMInterfaceAgentTrace: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 ip_address = 10.0.0.17 virtual_network = default-domain:default-project:ip-fabric mac_address = fa:16:3e:42:60:74 label = 17 active = 1 vm_name = l2_active = 1 uuid = 6fd2c88c-dcd0-4807-b0f8-05d81d30aac7 ip6_address = :: ip6_active = 1 floating_ips= [ [ ] ] vm_uuid = ip4_active = 1 is_health_check_active = 1 health_check_instance_list= [ [ ] ] admin_state = 1 fixed_ip4_list= [ [ (*_iter26) = 10.0.0.17, ] ] fixed_ip6_list= [ [ ] ] vn_uuid = 2d6b6d1c-978e-4fe6-9af3-aa814e83dd99 tx_vlan = 65535 rx_vlan = 65535 parent_interface = ens3 vhostuser_mode = 0 port_mirror_enabled = 0 ] 2025-02-13 Thu 02:19:21:494.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVMInterfaceAgentTrace 2025-02-13 Thu 02:19:21:494.503 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:494.516 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161494423 [SYS_NOTICE]: UVEVMITags: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 app = tier = site = deployment = labels= [ [ ] ] custom_tags= [ [ ] ] ] 2025-02-13 Thu 02:19:21:494.523 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMITags 2025-02-13 Thu 02:19:21:494.531 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:494.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161494461 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 connected_networks= [ [ (*_iter142) = default-domain:default-project:ip-fabric, ] ] vn_count = 1 ] 2025-02-13 Thu 02:19:21:494.550 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:19:21:494.559 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:494.572 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161494484 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 interface_list= [ [ (*_iter143) = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0, ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ ] ] vmi_count= [ active = 1 ] down_interface_count = 0 ] 2025-02-13 Thu 02:19:21:494.578 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:19:21:496.112 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:500.307 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413161496081 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric associated_fip_count = 0 in_bandwidth_usage = 0 out_bandwidth_usage = 0 vrf_stats_list= [ [ [ name = default-domain:default-project:ip-fabric:ip-fabric diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:19:21:500.345 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:19:21:500.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:500.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413161496100 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2025-02-13 Thu 02:19:21:500.487 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:19:21:500.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:500.597 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413161496212 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric port_bucket_bmap= [ udp_sport_bitmap= [ [ (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, (*_iter24) = 0, ] ] udp_dport_bitmap= [ [ (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, (*_iter25) = 0, ] ] tcp_sport_bitmap= [ [ (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, (*_iter26) = 0, ] ] tcp_dport_bitmap= [ [ (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, (*_iter27) = 0, ] ] ] raw_if_stats= [ in_pkts = 51 in_bytes = 3120 out_pkts = 55 out_bytes = 2959 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 0 out_bw_usage = 0 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:19:21:500.611 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:19:21:500.619 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:500.635 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413161496248 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:19:21:500.642 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:19:21:729.898 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:730.073 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413161729618 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 3 dequeues = 3 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 56 dequeues = 56 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 69 dequeues = 69 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413153733779 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413161500618 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:19:21:730.102 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:21:730.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:730.397 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413161729793 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 21, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 56, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:21:730.427 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:21:731.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:732.131 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161731937 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 xmpp_peer_list= [ [ [ ip = 10.0.0.17 status = 1 setup_time = 1739413131818652 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.17, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2025-02-13 00:26:33.389417","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-7769-1.novalocal","build-id":"unknown","build-number":"unknown"}]} phy_if= [ [ [ name = ens3 mac_address = fa:16:3e:42:60:74 ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:42:60:74 ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_DEBUG log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.17, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.17 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = ens3, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.17 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.17:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.17:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-juju-os-473-1 flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.17:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2025-02-13 00:23:54.879368", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-7769-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2025-02-13 Thu 02:19:21:732.166 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVrouterAgent 2025-02-13 Thu 02:19:21:732.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:732.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161731992 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 51 in_bytes = 3120 out_tpkts = 55 out_bytes = 2959 exception_packets = 27 exception_packets_dropped = 11 exception_packets_allowed = 16 total_flows = 0 aged_flows = 0 uptime = 1739413131494164 phy_if_info= [ [ _iter254->first = ens3 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 51 in_bytes = 3120 out_pkts = 55 out_bytes = 2959 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 25 in_bytes = 1699 out_pkts = 54 out_bytes = 3205 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:19:21:732.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:19:21:732.488 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:21:732.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413161732035 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:19:21:732.547 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:19:21:733.904 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:19:21:734.202 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:19:21:734.351 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:21:734.383 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413161733963 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:21:734.392 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:21:734.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:21:734.640 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413161734337 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 21, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 61, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:21:734.664 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:37:734.319 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:37:734.411 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:19:37:734.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:19:37:734.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Idle 2025-02-13 Thu 02:19:37:734.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:19:37:734.691 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:37:734.729 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413177734494 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:37:734.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:37:734.750 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:19:37:734.807 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:37:735.084 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413177734676 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 22, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 63, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:37:735.112 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:49:201.813 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33926 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:19:49:250.912 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33926 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:19:51:495.772 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:495.838 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413191495736 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:19:51:495.851 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:19:51:495.904 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:495.989 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413191495895 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 77 in_bytes = 4605 out_pkts = 82 out_bytes = 4406 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 409 out_bw_usage = 399 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:19:51:496.001 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:19:51:496.012 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:496.038 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413191495903 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:19:51:496.046 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:19:51:730.135 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:730.229 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413191729943 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 68 dequeues = 68 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 84 dequeues = 84 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413177734572 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413191496010 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:19:51:730.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:51:730.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:730.518 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413191730096 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 24, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 68, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:51:730.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:19:51:732.879 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:732.946 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413191732824 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.4975 five_min_avg = 0.61 fifteen_min_avg = 0.43625 ] cpu_share = 0.474289 ] ] 2025-02-13 Thu 02:19:51:732.957 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:19:51:732.966 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:732.984 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413191732874 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.474289 used_sys_mem = 4269468888 one_min_cpuload = 0.4975 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:19:51:732.992 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:19:51:733.001 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:733.157 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413191732993 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 77 in_bytes = 4605 out_tpkts = 82 out_bytes = 4406 exception_packets = 28 exception_packets_allowed = 17 raw_vhost_stats= [ name = vhost0 in_pkts = 77 in_bytes = 4605 out_pkts = 82 out_bytes = 4406 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 37 in_bytes = 2516 out_pkts = 80 out_bytes = 4690 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:19:51:733.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:19:51:733.193 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:19:51:733.222 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413191733041 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:19:51:733.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:19:53:734.686 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:19:53:735.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:19:53:735.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:53:735.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413193734785 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:19:53:735.358 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:19:53:735.368 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:19:53:735.641 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413193735302 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 24, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 74, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:19:53:735.663 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:21:496.060 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:496.133 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413221495993 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:20:21:496.146 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:20:21:496.207 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:496.374 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413221496177 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 98 in_bytes = 5695 out_pkts = 102 out_bytes = 5318 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 290 out_bw_usage = 243 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:20:21:496.416 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:20:21:496.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:496.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413221496188 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:20:21:496.459 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:20:21:730.461 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:730.598 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413221730299 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 79 dequeues = 79 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 96 dequeues = 96 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413193734796 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413221496429 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:20:21:730.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:21:730.626 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:731.056 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413221730403 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 24, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 79, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:20:21:731.086 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:21:733.259 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:733.453 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413221733225 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 98 in_bytes = 5695 out_tpkts = 102 out_bytes = 5318 exception_packets = 30 exception_packets_dropped = 12 exception_packets_allowed = 18 raw_vhost_stats= [ name = vhost0 in_pkts = 98 in_bytes = 5695 out_pkts = 102 out_bytes = 5318 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.91468e-16 total_out_bandwidth_utilization = 1.85941e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 146, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 343, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 42 in_bytes = 2798 out_pkts = 101 out_bytes = 5780 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:20:21:733.480 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:20:21:733.491 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:21:733.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413221733315 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:20:21:733.546 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:20:23:735.338 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:20:23:735.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:20:23:735.469 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:20:23:735.592 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:20:23:735.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > SSL Handshake failed due to error: 125 category: system message: Operation canceled src/contrail-common/io/ssl_server.cc 134 2025-02-13 Thu 02:20:23:735.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:23:735.746 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413223735505 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:20:23:735.762 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:20:23:735.771 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:20:23:735.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:23:736.177 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413223735697 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 83, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:20:23:736.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:51:496.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:496.228 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413251496130 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:20:51:496.239 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:20:51:496.248 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:496.317 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413251496232 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 219 in_bytes = 15949 out_pkts = 225 out_bytes = 15604 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2734 out_bw_usage = 2742 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:20:51:496.343 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:20:51:496.352 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:496.363 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413251496236 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:20:51:496.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:20:51:730.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:730.860 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413251730588 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 88 dequeues = 88 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 107 dequeues = 107 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413223735589 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413251496350 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:20:51:730.875 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:51:730.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:731.173 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413251730726 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 88, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:20:51:731.203 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:20:51:733.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:733.608 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413251733397 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.92625 five_min_avg = 0.68625 fifteen_min_avg = 0.47125 ] cpu_share = 0.441667 ] ] 2025-02-13 Thu 02:20:51:733.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:20:51:733.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:733.671 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413251733409 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.441667 used_sys_mem = 4269468888 one_min_cpuload = 0.92625 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:20:51:733.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:20:51:733.687 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:733.851 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413251733437 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 219 in_bytes = 15949 out_tpkts = 225 out_bytes = 15604 exception_packets = 31 exception_packets_allowed = 19 raw_vhost_stats= [ name = vhost0 in_pkts = 219 in_bytes = 15949 out_pkts = 225 out_bytes = 15604 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 150 in_bytes = 12454 out_pkts = 222 out_bytes = 16034 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:20:51:733.872 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:20:51:733.883 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:20:51:733.913 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413251733461 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:20:51:733.920 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:20:53:735.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:20:53:735.987 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:20:53:736.122 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:53:736.155 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413253735802 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:20:53:736.175 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:20:53:736.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:20:53:736.439 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413253736102 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 94, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:20:53:736.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:21:498.078 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:498.160 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413281497974 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:21:21:498.175 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:21:21:498.188 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:498.266 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413281498178 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 245 in_bytes = 17565 out_pkts = 251 out_bytes = 17148 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 430 out_bw_usage = 411 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:21:21:498.294 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:21:21:498.303 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:498.315 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413281498185 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:21:21:498.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:21:21:731.103 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:731.231 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413281730924 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 99 dequeues = 99 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 119 dequeues = 119 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413253735811 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413281498302 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:21:21:731.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:21:731.256 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:731.500 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413281731061 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 99, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:21:21:732.359 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:21:733.703 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:733.902 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413281733670 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 245 in_bytes = 17565 out_tpkts = 251 out_bytes = 17148 exception_packets = 32 exception_packets_dropped = 13 raw_vhost_stats= [ name = vhost0 in_pkts = 245 in_bytes = 17565 out_pkts = 251 out_bytes = 17148 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.6382e-15 total_out_bandwidth_utilization = 8.57604e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1409, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1582, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 161 in_bytes = 13368 out_pkts = 248 out_bytes = 17650 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:21:21:733.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:21:21:733.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:21:733.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413281733718 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:21:21:733.983 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:21:23:736.170 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:21:23:736.280 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:21:23:736.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:21:23:736.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:21:23:736.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:23:736.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413283736373 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:21:23:736.600 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:21:23:736.643 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:21:23:736.684 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:23:736.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413283736534 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 103, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:21:23:736.958 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:51:498.226 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:498.301 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413311498188 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:21:51:498.314 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:21:51:498.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:498.578 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413311498374 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 267 in_bytes = 18717 out_pkts = 273 out_bytes = 18192 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 307 out_bw_usage = 278 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:21:51:498.599 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:21:51:498.632 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:498.650 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413311498384 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:21:51:498.662 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:21:51:731.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:731.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413311731211 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 108 dequeues = 108 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 130 dequeues = 130 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413283736442 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413311498629 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:21:51:731.531 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:51:731.540 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:731.799 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413311731401 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 108, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:21:51:731.829 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:21:51:733.883 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:733.952 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413311733854 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.51875 five_min_avg = 0.61125 fifteen_min_avg = 0.45875 ] cpu_share = 0.26875 ] ] 2025-02-13 Thu 02:21:51:734.008 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:21:51:734.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:734.115 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413311733867 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.26875 used_sys_mem = 4269468888 one_min_cpuload = 0.51875 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:21:51:734.123 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:21:51:734.131 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:734.278 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413311733907 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 267 in_bytes = 18717 out_tpkts = 273 out_bytes = 18192 exception_packets = 33 exception_packets_allowed = 20 raw_vhost_stats= [ name = vhost0 in_pkts = 267 in_bytes = 18717 out_pkts = 273 out_bytes = 18192 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 168 in_bytes = 13782 out_pkts = 270 out_bytes = 18802 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:21:51:734.295 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:21:51:734.305 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:21:51:734.334 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413311733933 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:21:51:734.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:21:53:736.587 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:21:53:736.835 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:21:53:736.943 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:53:736.988 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413313736672 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:21:53:736.997 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:21:53:737.018 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:21:53:737.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413313736932 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 114, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:21:53:737.333 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:00:062.406 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34338 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:22:00:107.229 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34338 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:22:21:498.281 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:498.373 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413341498232 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:22:21:498.386 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:22:21:498.441 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:498.560 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413341498414 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 292 in_bytes = 20140 out_pkts = 297 out_bytes = 19437 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 379 out_bw_usage = 332 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:22:21:498.590 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:22:21:498.601 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:498.614 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413341498423 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:22:21:498.621 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:22:21:731.743 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:731.900 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413341731578 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 119 dequeues = 119 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 142 dequeues = 142 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413313736680 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413341498599 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:22:21:731.917 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:21:731.927 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:732.232 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413341731693 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 119, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:22:21:732.265 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:21:734.138 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:734.331 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413341734099 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 292 in_bytes = 20140 out_tpkts = 297 out_bytes = 19437 exception_packets = 34 exception_packets_allowed = 21 raw_vhost_stats= [ name = vhost0 in_pkts = 292 in_bytes = 20140 out_pkts = 297 out_bytes = 19437 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.42678e-16 total_out_bandwidth_utilization = 1.85941e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 137, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 343, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 177 in_bytes = 14397 out_pkts = 295 out_bytes = 20225 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:22:21:734.357 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:22:21:734.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:21:734.401 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413341734142 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:22:21:734.409 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:22:23:736.990 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:22:23:737.085 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:22:23:737.125 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:22:23:737.243 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Idle 2025-02-13 Thu 02:22:23:737.340 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:23:737.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413343737181 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:22:23:737.379 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:22:23:737.403 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:22:23:737.443 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:23:737.700 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413343737329 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 123, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:22:23:737.723 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:51:498.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:498.617 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413371498502 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:22:51:498.630 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:22:51:498.753 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:498.858 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413371498728 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 314 in_bytes = 21292 out_pkts = 321 out_bytes = 20621 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 307 out_bw_usage = 315 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:22:51:498.873 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:22:51:498.884 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:498.911 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413371498735 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:22:51:498.919 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:22:51:732.028 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:732.118 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413371731835 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 128 dequeues = 128 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 153 dequeues = 153 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413343737241 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413371498883 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:22:51:732.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:51:732.142 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:732.398 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413371731983 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 128, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:22:51:732.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:22:51:734.396 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:734.467 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413371734364 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.33375 five_min_avg = 0.54125 fifteen_min_avg = 0.44375 ] cpu_share = 0.274954 ] ] 2025-02-13 Thu 02:22:51:734.478 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:22:51:734.499 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:734.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413371734391 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.274954 used_sys_mem = 4269468888 one_min_cpuload = 0.33375 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:22:51:734.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:22:51:734.532 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:734.693 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413371734425 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 314 in_bytes = 21292 out_tpkts = 321 out_bytes = 20621 exception_packets = 35 exception_packets_allowed = 22 raw_vhost_stats= [ name = vhost0 in_pkts = 314 in_bytes = 21292 out_pkts = 321 out_bytes = 20621 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 186 in_bytes = 14951 out_pkts = 317 out_bytes = 21377 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:22:51:734.709 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:22:51:734.720 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:22:51:734.751 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413371734469 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:22:51:734.758 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:22:53:737.389 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:22:53:737.634 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:22:53:737.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:53:737.774 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413373737476 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:22:53:737.783 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:22:53:737.791 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:22:53:738.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413373737732 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 16 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 27, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 134, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:22:53:738.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:00:330.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:37224 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:23:00:374.917 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:37224 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:23:11:455.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40456 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:23:11:499.014 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40456 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:23:21:498.601 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:498.675 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413401498521 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:23:21:498.689 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:23:21:498.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:498.796 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413401498708 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 343 in_bytes = 23020 out_pkts = 351 out_bytes = 22311 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 476 out_bw_usage = 466 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:23:21:498.822 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:23:21:498.831 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:498.843 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413401498723 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:23:21:498.850 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:23:21:732.419 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:732.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413401732258 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 139 dequeues = 139 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 165 dequeues = 165 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413373737484 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413401498830 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:23:21:732.593 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:21:732.604 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:732.846 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413401732374 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 139, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:23:21:732.874 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:21:734.691 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:736.038 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413401734659 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 343 in_bytes = 23020 out_tpkts = 351 out_bytes = 22311 exception_packets = 37 exception_packets_dropped = 14 exception_packets_allowed = 23 raw_vhost_stats= [ name = vhost0 in_pkts = 343 in_bytes = 23020 out_pkts = 351 out_bytes = 22311 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 1.16552e-15 total_out_bandwidth_utilization = 2.08167e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 215, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 384, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 201 in_bytes = 16011 out_pkts = 346 out_bytes = 23105 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:23:21:736.069 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:23:21:736.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:21:736.110 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413401734708 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:23:21:736.118 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:23:23:737.796 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:23:23:737.874 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:23:23:737.888 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:23:23:738.005 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Idle 2025-02-13 Thu 02:23:23:738.213 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:23:738.255 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413403737930 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:23:23:738.279 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:23:23:738.295 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:23:23:738.333 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:23:738.762 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413403738191 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 143, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:23:23:738.798 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:51:498.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:498.610 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413431498508 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:23:51:498.621 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:23:51:498.744 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:498.846 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413431498723 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 354 in_bytes = 23710 out_pkts = 362 out_bytes = 22893 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 184 out_bw_usage = 155 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:23:51:498.876 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:23:51:498.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:498.898 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413431498733 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:23:51:498.906 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:23:51:732.685 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:732.782 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413431732513 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 8 dequeues = 8 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 148 dequeues = 148 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 176 dequeues = 176 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413403738003 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413431498884 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:23:51:732.797 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:51:732.807 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:733.094 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413431732640 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 148, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:23:51:733.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:23:51:734.844 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:734.908 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413431734822 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.27625 five_min_avg = 0.48875 fifteen_min_avg = 0.4325 ] cpu_share = 0.275 ] ] 2025-02-13 Thu 02:23:51:734.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:23:51:734.939 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:734.956 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413431734835 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.275 used_sys_mem = 4269468888 one_min_cpuload = 0.27625 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:23:51:734.964 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:23:51:734.971 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:735.122 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413431734865 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 354 in_bytes = 23710 out_tpkts = 362 out_bytes = 22893 raw_vhost_stats= [ name = vhost0 in_pkts = 354 in_bytes = 23710 out_pkts = 362 out_bytes = 22893 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 206 in_bytes = 16341 out_pkts = 357 out_bytes = 23795 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:23:51:735.138 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:23:51:735.146 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:23:51:735.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413431734889 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:23:51:735.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:23:53:738.169 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:23:53:738.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:23:53:738.535 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:53:738.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413433738267 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:23:53:738.573 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:23:53:738.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:23:53:738.840 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413433738519 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 154, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:23:53:738.861 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:04:697.645 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41564 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:24:04:742.914 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:41564 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:24:21:498.730 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:498.819 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413461498692 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:24:21:498.832 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:24:21:498.842 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:498.927 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413461498834 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 368 in_bytes = 24643 out_pkts = 378 out_bytes = 23802 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 248 out_bw_usage = 242 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:24:21:498.954 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:24:21:498.963 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:498.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413461498842 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:24:21:498.983 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:24:21:732.945 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:733.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413461732825 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 159 dequeues = 159 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 188 dequeues = 188 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413433738274 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413461498962 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:24:21:733.144 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:21:733.155 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:733.409 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413461732912 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 159, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:24:21:733.446 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:21:735.106 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:735.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413461735067 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 368 in_bytes = 24643 out_tpkts = 378 out_bytes = 23802 exception_packets = 41 exception_packets_allowed = 27 raw_vhost_stats= [ name = vhost0 in_pkts = 368 in_bytes = 24643 out_pkts = 378 out_bytes = 23802 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 411 out_bandwidth_usage = 576 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 8.0231e-16 total_out_bandwidth_utilization = 1.23057e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 148, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 227, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 219 in_bytes = 17124 out_pkts = 373 out_bytes = 24812 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:24:21:735.478 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:24:21:735.506 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:21:735.556 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413461735103 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:24:21:735.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:24:23:738.599 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:24:23:738.703 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:24:23:738.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:24:23:738.865 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:24:23:739.008 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:23:739.067 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413463738798 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:24:23:739.081 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:24:23:739.094 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:24:23:739.154 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:23:739.411 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413463738990 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 163, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:24:23:739.436 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:51:500.364 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:500.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413491500315 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:24:51:500.442 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:24:51:500.474 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:500.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413491500464 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 375 in_bytes = 25117 out_pkts = 385 out_bytes = 24168 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 97 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:24:51:500.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:24:51:500.578 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:500.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413491500475 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:24:51:500.598 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:24:51:733.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:733.294 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413491733011 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 9 dequeues = 9 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 168 dequeues = 168 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 199 dequeues = 199 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413463738864 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413491500577 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:24:51:733.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:51:733.318 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:733.577 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413491733144 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 168, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:24:51:733.610 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:24:51:735.265 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:735.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413491735235 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.215 five_min_avg = 0.4325 fifteen_min_avg = 0.41625 ] cpu_share = 0.272917 ] ] 2025-02-13 Thu 02:24:51:735.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:24:51:735.377 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:735.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413491735246 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.272917 used_sys_mem = 4269468888 one_min_cpuload = 0.215 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:24:51:735.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:24:51:735.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:735.573 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413491735293 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 375 in_bytes = 25117 out_tpkts = 385 out_bytes = 24168 raw_vhost_stats= [ name = vhost0 in_pkts = 375 in_bytes = 25117 out_pkts = 385 out_bytes = 24168 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 223 in_bytes = 17364 out_pkts = 380 out_bytes = 25286 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:24:51:735.590 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:24:51:735.600 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:24:51:735.631 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413491735320 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:24:51:735.638 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:24:53:739.028 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:24:53:739.272 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:24:53:739.377 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:53:739.422 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413493739110 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:24:53:739.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:24:53:739.439 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:24:53:739.691 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413493739364 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 174, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:24:53:740.486 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:21:500.778 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:500.859 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413521500689 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:25:21:500.872 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:25:21:500.884 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:500.976 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413521500849 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 382 in_bytes = 25591 out_pkts = 395 out_bytes = 24744 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] out_bw_usage = 153 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:25:21:500.990 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:25:21:500.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:501.011 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413521500857 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:25:21:501.487 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:25:21:733.490 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:733.647 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413521733332 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 179 dequeues = 179 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 211 dequeues = 211 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413493739117 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413521500998 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:25:21:733.666 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:21:733.676 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:733.949 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413521733428 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 179, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:25:21:733.986 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:21:735.538 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:735.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413521735513 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 382 in_bytes = 25591 out_tpkts = 395 out_bytes = 24744 raw_vhost_stats= [ name = vhost0 in_pkts = 382 in_bytes = 25591 out_pkts = 395 out_bytes = 24744 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 4.98733e-16 total_out_bandwidth_utilization = 6.83047e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 92, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 126, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 230 in_bytes = 17814 out_pkts = 387 out_bytes = 25760 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:25:21:735.744 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:25:21:735.759 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:21:735.790 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413521735555 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:25:21:735.797 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:25:23:739.453 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:25:23:739.560 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:25:23:739.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:25:23:739.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Idle 2025-02-13 Thu 02:25:23:739.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:23:739.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413523739664 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:25:23:739.915 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:25:23:739.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:25:23:739.977 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:23:740.297 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413523739836 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 183, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:25:23:740.347 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:51:500.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:501.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551500915 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:25:51:501.046 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:25:51:501.173 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:501.250 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551501160 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 464 in_bytes = 32746 out_pkts = 477 out_bytes = 31791 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 1908 out_bw_usage = 1879 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:25:51:501.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:25:51:501.271 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:501.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551501164 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:25:51:501.291 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:25:51:733.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:733.939 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413551733587 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 188 dequeues = 188 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 222 dequeues = 222 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413523739740 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413551501269 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:25:51:733.974 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:51:733.991 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:734.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413551733738 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 188, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:25:51:734.457 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:25:51:735.761 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:735.838 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551735731 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.725 five_min_avg = 0.50875 fifteen_min_avg = 0.44125 ] cpu_share = 0.2875 ] ] 2025-02-13 Thu 02:25:51:735.854 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:25:51:735.869 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:735.893 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551735749 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.2875 used_sys_mem = 4269468888 one_min_cpuload = 0.725 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:25:51:735.905 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:25:51:735.927 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:736.188 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551735784 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 464 in_bytes = 32746 out_tpkts = 477 out_bytes = 31791 exception_packets = 43 exception_packets_allowed = 29 raw_vhost_stats= [ name = vhost0 in_pkts = 464 in_bytes = 32746 out_pkts = 477 out_bytes = 31791 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 309 in_bytes = 24735 out_pkts = 469 out_bytes = 32915 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:25:51:736.216 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:25:51:736.229 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:25:51:736.276 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413551735820 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:25:51:736.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:25:53:739.884 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:25:53:740.137 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:25:53:740.267 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:53:740.299 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413553739964 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:25:53:740.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:25:53:740.319 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:25:53:740.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413553740253 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 194, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:25:53:740.594 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:21:501.194 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:501.253 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413581501141 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:26:21:501.266 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:26:21:501.337 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:501.480 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413581501308 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 510 in_bytes = 36799 out_pkts = 522 out_bytes = 35666 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 1080 out_bw_usage = 1033 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:26:21:501.503 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:26:21:501.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:501.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413581501315 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:26:21:501.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:26:21:734.067 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:734.185 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413581733922 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 199 dequeues = 199 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 234 dequeues = 234 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413553739973 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413581501515 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:26:21:734.215 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:21:734.224 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:734.471 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413581734008 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 199, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:26:21:734.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:21:735.982 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:736.196 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413581735953 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 510 in_bytes = 36799 out_tpkts = 522 out_bytes = 35666 raw_vhost_stats= [ name = vhost0 in_pkts = 510 in_bytes = 36799 out_pkts = 522 out_bytes = 35666 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.70868e-15 total_out_bandwidth_utilization = 8.09899e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1422, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1494, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 351 in_bytes = 28484 out_pkts = 515 out_bytes = 36968 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:26:21:736.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:26:21:736.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:21:736.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413581735992 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:26:21:737.163 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:26:23:740.313 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:26:23:740.407 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:26:23:740.418 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:26:23:740.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Idle 2025-02-13 Thu 02:26:23:740.652 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:23:740.681 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413583740461 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:26:23:740.690 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:26:23:740.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:26:23:740.740 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:23:740.990 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413583740640 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 203, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:26:23:741.041 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:51:502.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:502.830 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413611502689 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:26:51:502.857 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:26:51:502.950 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:503.065 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413611502924 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 521 in_bytes = 37537 out_pkts = 535 out_bytes = 36436 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 196 out_bw_usage = 205 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:26:51:503.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:26:51:503.091 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:503.103 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413611502968 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:26:51:503.111 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:26:51:734.396 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:734.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413611734150 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 11 dequeues = 11 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 208 dequeues = 208 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 245 dequeues = 245 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413583740540 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413611503090 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:26:51:734.505 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:51:734.514 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:734.809 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413611734354 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 208, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:26:51:734.833 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:51:736.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:736.255 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413611736152 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.4225 five_min_avg = 0.46 fifteen_min_avg = 0.42875 ] cpu_share = 0.266667 ] ] 2025-02-13 Thu 02:26:51:736.269 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:26:51:736.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:736.301 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413611736162 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.266667 used_sys_mem = 4269468888 one_min_cpuload = 0.4225 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:26:51:736.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:26:51:736.316 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:736.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413611736192 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 521 in_bytes = 37537 out_tpkts = 535 out_bytes = 36436 exception_packets = 44 exception_packets_allowed = 30 raw_vhost_stats= [ name = vhost0 in_pkts = 521 in_bytes = 37537 out_pkts = 535 out_bytes = 36436 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 361 in_bytes = 29128 out_pkts = 526 out_bytes = 37706 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:26:51:736.522 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:26:51:736.535 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:26:51:736.581 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413611736226 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:26:51:736.595 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:26:53:740.691 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:26:53:740.943 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:26:53:741.073 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:53:741.116 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413613740782 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:26:53:741.126 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:26:53:741.135 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:26:53:741.381 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413613741054 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 22 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 214, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:26:53:741.408 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:26:57:805.553 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:46478 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:26:57:850.927 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:46478 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:27:21:503.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:503.338 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413641503195 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:27:21:503.350 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:27:21:503.408 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:503.516 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413641503374 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 533 in_bytes = 38326 out_pkts = 549 out_bytes = 37293 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 210 out_bw_usage = 228 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:27:21:503.531 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:27:21:503.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:503.556 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413641503385 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:27:21:503.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:27:21:734.692 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:734.826 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413641734527 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 219 dequeues = 219 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 257 dequeues = 257 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413613740791 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413641503541 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:27:21:734.858 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:27:21:734.868 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:735.192 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413641734642 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 59 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 219, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:27:21:735.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:27:21:736.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:736.639 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413641736376 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 533 in_bytes = 38326 out_tpkts = 549 out_bytes = 37293 exception_packets = 46 exception_packets_dropped = 15 exception_packets_allowed = 31 raw_vhost_stats= [ name = vhost0 in_pkts = 533 in_bytes = 38326 out_pkts = 549 out_bytes = 37293 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 9.92045e-16 total_out_bandwidth_utilization = 1.10047e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 183, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 203, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 372 in_bytes = 29859 out_pkts = 538 out_bytes = 38495 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:27:21:736.666 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:27:21:736.676 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:21:736.707 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413641736449 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:27:21:736.718 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:27:23:741.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:27:23:741.227 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:27:23:741.273 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:27:23:741.398 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Idle 2025-02-13 Thu 02:27:23:741.521 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:23:741.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413643741321 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:27:23:741.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:27:23:741.583 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:27:23:741.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:23:741.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413643741501 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 61 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 223, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:27:23:741.917 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:27:51:503.194 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:503.265 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413671503156 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:27:51:503.276 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:27:51:503.351 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:503.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413671503326 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 540 in_bytes = 38800 out_pkts = 558 out_bytes = 37799 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 130 out_bw_usage = 139 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:27:51:503.491 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:27:51:503.501 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:503.513 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413671503332 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:27:51:503.520 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:27:51:734.939 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:735.069 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413671734780 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 12 dequeues = 12 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 228 dequeues = 228 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 268 dequeues = 268 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413643741396 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413671503499 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:27:51:735.100 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:27:51:735.111 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:735.371 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413671734892 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 62 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 228, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:27:51:735.391 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:27:51:736.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:736.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413671736608 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.32 five_min_avg = 0.42 fifteen_min_avg = 0.4175 ] cpu_share = 0.270833 ] ] 2025-02-13 Thu 02:27:51:736.706 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:27:51:736.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:736.735 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413671736636 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.270833 used_sys_mem = 4269468888 one_min_cpuload = 0.32 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:27:51:736.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:27:51:736.749 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:736.894 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413671736672 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 540 in_bytes = 38800 out_tpkts = 558 out_bytes = 37799 raw_vhost_stats= [ name = vhost0 in_pkts = 540 in_bytes = 38800 out_pkts = 558 out_bytes = 37799 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 378 in_bytes = 30239 out_pkts = 545 out_bytes = 38969 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:27:51:736.916 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:27:51:736.924 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:27:51:736.954 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413671736704 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:27:51:737.085 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:27:53:741.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:27:53:741.795 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:27:53:741.900 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:53:741.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413673741620 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:27:53:741.942 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:27:53:741.951 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:27:53:742.293 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413673741888 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 64 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 35, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 234, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:27:53:742.334 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:12:230.238 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53726 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:28:12:275.034 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53726 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:28:21:503.382 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:503.451 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413701503326 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:28:21:503.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:28:21:503.540 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:503.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413701503517 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 556 in_bytes = 39865 out_pkts = 574 out_bytes = 38756 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 284 out_bw_usage = 255 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:28:21:503.664 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:28:21:503.673 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:503.689 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413701503524 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:28:21:503.696 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:28:21:735.254 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:735.407 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413701735084 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 239 dequeues = 239 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 280 dequeues = 280 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413673741629 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413701503672 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:28:21:735.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:21:735.447 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:735.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413701735200 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 65 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 26 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 37, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 239, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:28:21:735.769 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:21:736.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:737.101 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413701736865 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 556 in_bytes = 39865 out_tpkts = 574 out_bytes = 38756 exception_packets = 48 exception_packets_allowed = 33 raw_vhost_stats= [ name = vhost0 in_pkts = 556 in_bytes = 39865 out_pkts = 574 out_bytes = 38756 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 8.72783e-16 total_out_bandwidth_utilization = 1.11131e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 161, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 205, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 391 in_bytes = 31070 out_pkts = 561 out_bytes = 40034 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:28:21:737.131 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:28:21:737.143 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:21:737.174 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413701736912 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:28:21:737.181 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:28:23:399.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:36240 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:28:23:442.875 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:36240 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:28:23:741.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:28:23:742.072 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:28:23:742.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:28:23:742.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:28:23:742.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:23:742.339 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413703742159 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:28:23:742.348 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:28:23:742.358 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:28:23:742.389 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:23:742.661 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413703742296 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 67 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 28 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 39, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 243, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:28:23:742.693 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:51:503.378 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:503.449 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413731503336 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:28:51:503.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:28:51:503.489 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:503.586 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413731503464 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 567 in_bytes = 40672 out_pkts = 585 out_bytes = 39455 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 222 out_bw_usage = 192 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:28:51:503.602 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:28:51:503.614 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:503.631 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413731503468 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:28:51:503.640 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:28:51:735.485 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:735.607 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413731735316 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 248 dequeues = 248 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 291 dequeues = 291 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413703742209 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413731503612 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:28:51:735.657 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:51:735.668 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:735.955 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413731735452 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 68 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 28 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 39, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 248, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:28:51:735.979 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:28:51:737.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:737.163 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413731737051 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.2375 five_min_avg = 0.3775 fifteen_min_avg = 0.4025 ] cpu_share = 0.275 ] ] 2025-02-13 Thu 02:28:51:737.179 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:28:51:737.193 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:737.218 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413731737063 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.275 used_sys_mem = 4269468888 one_min_cpuload = 0.2375 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:28:51:737.228 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:28:51:737.239 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:737.478 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413731737093 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 567 in_bytes = 40672 out_tpkts = 585 out_bytes = 39455 raw_vhost_stats= [ name = vhost0 in_pkts = 567 in_bytes = 40672 out_pkts = 585 out_bytes = 39455 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 399 in_bytes = 31643 out_pkts = 572 out_bytes = 40841 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:28:51:737.509 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:28:51:737.522 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:28:51:737.570 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413731737127 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:28:51:737.587 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:28:53:742.353 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:28:53:742.573 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:28:53:742.673 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:53:742.721 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413733742438 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:28:53:742.730 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:28:53:742.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:28:53:743.007 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413733742662 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 28 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 39, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 254, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:28:53:743.051 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:07:622.808 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34240 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:29:07:666.962 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:34240 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:29:21:504.997 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:505.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413761504958 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:29:21:505.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:29:21:505.103 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:505.180 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413761505094 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 580 in_bytes = 41563 out_pkts = 600 out_bytes = 40322 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 237 out_bw_usage = 231 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:29:21:505.195 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:29:21:505.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:505.218 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413761505101 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:29:21:505.225 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:29:21:735.785 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:735.923 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413761735634 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 259 dequeues = 259 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 303 dequeues = 303 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413733742445 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413761505205 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:29:21:735.992 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:21:736.008 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:736.285 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413761735737 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 259, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:29:21:736.310 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:21:737.313 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:737.598 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413761737268 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 580 in_bytes = 41563 out_tpkts = 600 out_bytes = 40322 exception_packets = 51 exception_packets_allowed = 36 raw_vhost_stats= [ name = vhost0 in_pkts = 580 in_bytes = 41563 out_pkts = 600 out_bytes = 40322 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 406 out_bandwidth_usage = 453 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 9.48677e-16 total_out_bandwidth_utilization = 1.28478e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 175, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 237, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 411 in_bytes = 32384 out_pkts = 587 out_bytes = 41816 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:29:21:737.633 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:29:21:737.645 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:21:737.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413761737315 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:29:21:737.688 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:29:23:742.729 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:29:23:742.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:29:23:742.834 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:29:23:742.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Idle 2025-02-13 Thu 02:29:23:743.087 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:23:743.122 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413763742874 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:29:23:743.130 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:29:23:743.139 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:29:23:743.176 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:23:743.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413763743065 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 263, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:29:23:744.381 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:51:505.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:505.315 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413791505193 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:29:51:505.341 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:29:51:505.389 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:505.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413791505363 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 593 in_bytes = 42481 out_pkts = 623 out_bytes = 41988 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 244 out_bw_usage = 444 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:29:51:505.527 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:29:51:505.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:505.550 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413791505367 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:29:51:505.558 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:29:51:736.002 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:736.102 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413791735848 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 14 dequeues = 14 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 268 dequeues = 268 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 314 dequeues = 314 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413763742938 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413791505536 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:29:51:736.141 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:51:736.151 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:736.394 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413791735966 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 268, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:29:51:736.413 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:29:51:737.484 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:737.602 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413791737430 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.2275 five_min_avg = 0.35 fifteen_min_avg = 0.39125 ] cpu_share = 0.275 ] ] 2025-02-13 Thu 02:29:51:737.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:29:51:737.635 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:737.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413791737458 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.275 used_sys_mem = 4269468888 one_min_cpuload = 0.2275 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:29:51:737.663 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:29:51:737.671 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:737.857 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413791737488 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 593 in_bytes = 42481 out_tpkts = 623 out_bytes = 41988 exception_packets = 52 exception_packets_allowed = 37 raw_vhost_stats= [ name = vhost0 in_pkts = 593 in_bytes = 42481 out_pkts = 623 out_bytes = 41988 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 431 in_bytes = 33924 out_pkts = 600 out_bytes = 42734 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:29:51:737.877 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:29:51:737.889 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:29:51:737.919 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413791737513 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:29:51:737.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:29:53:743.137 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:29:53:743.437 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:29:53:743.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:53:743.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413793743242 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:29:53:743.631 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:29:53:743.643 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:29:53:743.971 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413793743559 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 274, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:29:53:744.011 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:21:505.355 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:505.414 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413821505319 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:30:21:505.426 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:30:21:505.463 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:505.545 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413821505456 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 693 in_bytes = 51328 out_pkts = 726 out_bytes = 50937 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2359 out_bw_usage = 2386 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:30:21:505.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:30:21:505.566 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:505.577 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413821505462 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:30:21:505.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:30:21:736.359 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:736.549 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413821736179 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 279 dequeues = 279 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 326 dequeues = 326 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413793743253 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413821505564 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:30:21:736.587 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:21:736.607 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:737.069 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413821736298 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 279, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:30:21:737.101 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:21:737.741 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:738.064 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413821737707 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 693 in_bytes = 51328 out_tpkts = 726 out_bytes = 50937 exception_packets = 54 exception_packets_allowed = 39 raw_vhost_stats= [ name = vhost0 in_pkts = 693 in_bytes = 51328 out_pkts = 726 out_bytes = 50937 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.48642e-15 total_out_bandwidth_utilization = 7.05816e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1381, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1302, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 531 in_bytes = 42747 out_pkts = 700 out_bytes = 51581 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:30:21:738.116 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:30:21:738.129 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:21:738.161 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413821737757 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:30:21:738.169 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:30:23:743.572 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:30:23:743.649 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:30:23:743.692 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:30:23:743.816 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:30:23:743.928 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:23:743.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413823743733 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:30:23:743.982 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:30:23:743.991 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:30:23:744.043 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:23:744.295 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413823743915 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 283, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:30:23:744.329 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:51:505.547 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:505.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413851505510 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:30:51:505.625 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:30:51:505.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:505.848 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413851505682 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 726 in_bytes = 54207 out_pkts = 760 out_bytes = 53778 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 767 out_bw_usage = 757 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:30:51:505.879 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:30:51:505.889 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:505.902 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413851505687 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:30:51:505.910 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:30:51:736.576 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:736.671 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851736417 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 15 dequeues = 15 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 288 dequeues = 288 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 337 dequeues = 337 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413823743814 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413851505888 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:30:51:736.687 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:51:736.700 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:736.949 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851736529 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 288, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:30:51:736.980 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:30:51:737.912 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:737.956 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851737895 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.72625 five_min_avg = 0.49625 fifteen_min_avg = 0.44 ] cpu_share = 0.289583 ] ] 2025-02-13 Thu 02:30:51:737.965 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:30:51:738.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:738.065 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851737913 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.289583 used_sys_mem = 4269468888 one_min_cpuload = 0.72625 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:30:51:738.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:30:51:738.082 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:738.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851737953 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 726 in_bytes = 54207 out_tpkts = 760 out_bytes = 53778 raw_vhost_stats= [ name = vhost0 in_pkts = 726 in_bytes = 54207 out_pkts = 760 out_bytes = 53778 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 562 in_bytes = 45462 out_pkts = 733 out_bytes = 54460 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:30:51:738.239 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:30:51:738.249 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:30:51:738.279 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413851737986 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:30:51:738.286 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:30:53:744.057 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:30:53:744.303 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:30:53:744.426 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:53:744.458 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413853744146 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:30:53:744.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:30:53:744.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:30:53:744.769 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413853744412 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 294, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:30:53:744.794 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:21:505.703 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:505.779 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413881505660 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:31:21:505.795 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:31:21:505.890 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:505.994 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413881505869 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 743 in_bytes = 55371 out_pkts = 777 out_bytes = 55010 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 310 out_bw_usage = 328 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:31:21:506.051 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:31:21:506.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:506.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413881505879 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:31:21:506.081 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:31:21:736.922 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:737.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413881736722 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 299 dequeues = 299 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 349 dequeues = 349 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413853744154 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413881506060 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:31:21:737.096 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:21:737.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:737.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739413881736867 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 299, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:31:21:737.418 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:21:738.220 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:738.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413881738185 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 743 in_bytes = 55371 out_tpkts = 777 out_bytes = 55010 exception_packets = 56 exception_packets_allowed = 41 raw_vhost_stats= [ name = vhost0 in_pkts = 743 in_bytes = 55371 out_pkts = 777 out_bytes = 55010 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 2.75929e-15 total_out_bandwidth_utilization = 2.92192e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 509, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 539, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 576 in_bytes = 46568 out_pkts = 750 out_bytes = 55624 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:31:21:739.321 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:31:21:739.341 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:21:739.376 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413881738230 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:31:21:739.384 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:31:23:744.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:31:23:744.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:31:23:744.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:31:23:744.692 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:31:23:744.813 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:23:744.844 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413883744628 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:31:23:744.864 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:31:23:744.873 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:31:23:744.907 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:23:745.197 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413883744800 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 303, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:31:23:745.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:51:505.819 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:505.876 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413911505783 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:31:51:505.887 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:31:51:505.987 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:506.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413911505951 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 750 in_bytes = 55845 out_pkts = 785 out_bytes = 55446 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 126 out_bw_usage = 116 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:31:51:506.200 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:31:51:506.214 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:506.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739413911505968 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:31:51:506.245 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:31:51:737.173 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:737.268 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413911736991 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 308 dequeues = 308 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 360 dequeues = 360 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413883744690 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413911506212 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:31:51:737.282 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:51:737.292 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:737.556 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413911737122 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 25 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 308, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:31:51:737.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:31:51:738.453 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:738.502 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413911738437 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.45875 five_min_avg = 0.45875 fifteen_min_avg = 0.43 ] cpu_share = 0.266667 ] ] 2025-02-13 Thu 02:31:51:738.511 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:31:51:738.520 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:738.539 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413911738460 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.266667 used_sys_mem = 4269468888 one_min_cpuload = 0.45875 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:31:51:738.546 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:31:51:738.554 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:738.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413911738492 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 750 in_bytes = 55845 out_tpkts = 785 out_bytes = 55446 raw_vhost_stats= [ name = vhost0 in_pkts = 750 in_bytes = 55845 out_pkts = 785 out_bytes = 55446 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 581 in_bytes = 46878 out_pkts = 757 out_bytes = 56098 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:31:51:738.714 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:31:51:738.722 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:31:51:738.752 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413911738526 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:31:51:738.759 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:31:53:053.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:37702 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:31:53:098.920 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:37702 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:31:53:744.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:31:53:745.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:31:53:745.210 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:53:745.272 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413913744901 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:31:53:745.282 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:31:53:745.292 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:31:53:745.541 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413913745194 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 314, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:31:53:745.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:21:506.075 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:506.142 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413941505997 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:32:21:506.155 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:32:21:506.384 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:506.483 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413941506361 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 763 in_bytes = 56784 out_pkts = 799 out_bytes = 56347 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 250 out_bw_usage = 240 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:32:21:506.508 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:32:21:506.519 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:506.534 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413941506367 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:32:21:506.541 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:32:21:737.478 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:737.608 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413941737322 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 319 dequeues = 319 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 372 dequeues = 372 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413913744909 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739413941506517 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:32:21:737.626 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:21:737.644 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:737.899 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413941737426 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 26 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 319, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:32:21:737.937 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:21:738.707 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:738.907 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413941738678 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 763 in_bytes = 56784 out_tpkts = 799 out_bytes = 56347 exception_packets = 57 exception_packets_allowed = 42 raw_vhost_stats= [ name = vhost0 in_pkts = 763 in_bytes = 56784 out_pkts = 799 out_bytes = 56347 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.80626e-16 total_out_bandwidth_utilization = 1.01915e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 144, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 188, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 592 in_bytes = 47653 out_pkts = 770 out_bytes = 57037 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:32:21:738.932 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:32:21:738.942 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:21:738.981 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739413941738723 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:32:21:738.992 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:32:23:745.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:32:23:745.325 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:32:23:745.364 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:32:23:745.474 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Idle 2025-02-13 Thu 02:32:23:745.594 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:23:745.625 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413943745405 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:32:23:745.662 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:32:23:745.672 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:32:23:745.702 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:23:745.948 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739413943745567 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 323, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:32:23:745.970 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:51:507.651 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:507.706 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413971507596 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:32:51:507.718 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:32:51:508.378 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:508.532 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413971508338 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 772 in_bytes = 57342 out_pkts = 808 out_bytes = 56797 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 148 out_bw_usage = 120 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:32:51:508.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:32:51:508.578 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:508.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739413971508353 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:32:51:508.607 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:32:51:737.750 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:737.857 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413971737564 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 328 dequeues = 328 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 383 dequeues = 383 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739413943745473 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739413971508575 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:32:51:737.873 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:51:737.883 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:738.203 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413971737697 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 32 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 27 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 43, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 328, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:32:51:738.236 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:32:51:738.869 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:738.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413971738849 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.3425 five_min_avg = 0.42375 fifteen_min_avg = 0.41875 ] cpu_share = 0.260417 ] ] 2025-02-13 Thu 02:32:51:738.954 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:32:51:738.966 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:738.989 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413971738863 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.260417 used_sys_mem = 4269468888 one_min_cpuload = 0.3425 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:32:51:738.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:32:51:739.023 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:739.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413971738895 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 772 in_bytes = 57342 out_tpkts = 808 out_bytes = 56797 exception_packets = 58 exception_packets_allowed = 43 raw_vhost_stats= [ name = vhost0 in_pkts = 772 in_bytes = 57342 out_pkts = 808 out_bytes = 56797 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 598 in_bytes = 47977 out_pkts = 779 out_bytes = 57595 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:32:51:739.200 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:32:51:739.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:32:51:739.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739413971738928 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:32:51:739.247 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:32:51:953.848 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:51388 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:32:52:003.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:51388 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:32:53:745.634 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:32:53:745.920 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:32:53:746.057 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:53:746.103 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413973745724 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:32:53:746.125 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:32:53:746.135 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:32:53:746.395 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739413973746042 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 34 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 45, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 334, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:32:53:746.417 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:03:853.653 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40158 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:33:03:902.971 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:40158 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:33:21:509.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:509.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414001509412 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:33:21:509.556 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:33:21:509.607 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:509.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414001509582 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 789 in_bytes = 58554 out_pkts = 825 out_bytes = 57917 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 323 out_bw_usage = 298 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:33:21:509.787 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:33:21:509.798 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:509.811 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414001509589 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:33:21:509.819 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:33:21:738.318 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:738.493 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414001738112 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 339 dequeues = 339 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 395 dequeues = 395 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739413973745733 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414001509796 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:33:21:738.513 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:21:738.542 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:739.025 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414001738260 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 95 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 28 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 47, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 339, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:33:21:739.059 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:21:739.120 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:739.461 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414001739107 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 789 in_bytes = 58554 out_tpkts = 825 out_bytes = 57917 exception_packets = 59 exception_packets_allowed = 44 raw_vhost_stats= [ name = vhost0 in_pkts = 789 in_bytes = 58554 out_pkts = 825 out_bytes = 57917 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 9.48677e-16 total_out_bandwidth_utilization = 1.27936e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 175, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 236, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 612 in_bytes = 48971 out_pkts = 796 out_bytes = 58807 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:33:21:739.489 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:33:21:739.505 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:21:739.555 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414001739155 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:33:21:739.567 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:33:23:746.098 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:33:23:746.220 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:33:23:746.260 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:33:23:746.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:33:23:746.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:23:746.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414003746295 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:33:23:746.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:33:23:746.526 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:33:23:746.559 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:23:746.829 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414003746450 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 47, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 343, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:33:23:746.863 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:51:509.434 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:509.566 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414031509389 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:33:51:509.590 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:33:51:509.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:509.830 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414031509650 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 798 in_bytes = 59112 out_pkts = 836 out_bytes = 58527 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 148 out_bw_usage = 162 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:33:51:509.867 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:33:51:509.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:509.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414031509659 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:33:51:509.915 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:33:51:738.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:738.680 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414031738398 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 348 dequeues = 348 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 406 dequeues = 406 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414003746359 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414031509882 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:33:51:738.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:51:738.705 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:738.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414031738514 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 98 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 29 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 47, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 348, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:33:51:739.047 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:51:739.296 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:739.363 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414031739265 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.35125 five_min_avg = 0.41 fifteen_min_avg = 0.415 ] cpu_share = 0.3 ] ] 2025-02-13 Thu 02:33:51:739.374 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:33:51:739.385 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:739.402 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414031739274 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.3 used_sys_mem = 4269468888 one_min_cpuload = 0.35125 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:33:51:739.410 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:33:51:739.418 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:739.578 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414031739311 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 798 in_bytes = 59112 out_tpkts = 836 out_bytes = 58527 exception_packets = 60 exception_packets_allowed = 45 raw_vhost_stats= [ name = vhost0 in_pkts = 798 in_bytes = 59112 out_pkts = 836 out_bytes = 58527 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 620 in_bytes = 49455 out_pkts = 805 out_bytes = 59365 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:33:51:739.600 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:33:51:739.611 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:33:51:739.641 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414031739345 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:33:51:739.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:33:53:746.515 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:33:53:746.741 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:33:53:746.860 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:53:746.889 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414033746591 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:33:53:746.907 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:33:53:746.916 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:33:53:747.211 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414033746845 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 100 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 36 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 47, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 354, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:33:53:747.236 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:33:53:837.551 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33150 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:33:53:882.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:33150 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:34:21:509.682 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:509.772 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414061509629 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:34:21:509.789 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:34:21:509.830 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:509.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414061509800 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 812 in_bytes = 60073 out_pkts = 852 out_bytes = 59464 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 256 out_bw_usage = 249 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:34:21:509.962 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:34:21:509.973 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:509.985 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414061509811 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:34:21:509.992 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:34:21:738.914 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:739.091 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414061738695 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 359 dequeues = 359 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 418 dequeues = 418 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414033746599 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414061509971 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:34:21:739.110 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:34:21:739.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:739.561 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414061738852 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 101 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 359, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:34:21:739.614 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:34:21:739.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:739.866 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414061739506 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 812 in_bytes = 60073 out_tpkts = 852 out_bytes = 59464 exception_packets = 63 exception_packets_allowed = 48 raw_vhost_stats= [ name = vhost0 in_pkts = 812 in_bytes = 60073 out_pkts = 852 out_bytes = 59464 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] phy_if_5min_usage= [ [ [ name = ens3 in_bandwidth_usage = 476 out_bandwidth_usage = 495 ], ] ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 9.32414e-16 total_out_bandwidth_utilization = 1.15468e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 172, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 213, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 633 in_bytes = 50266 out_pkts = 821 out_bytes = 60410 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:34:21:739.884 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:34:21:739.894 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:21:739.924 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414061739552 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:34:21:739.932 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:34:23:746.897 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:34:23:746.986 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:34:23:747.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:34:23:747.141 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:34:23:747.264 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:23:747.294 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414063747075 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:34:23:747.312 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:34:23:747.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:34:23:747.352 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:23:747.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414063747251 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 103 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 363, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:34:23:747.636 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:34:51:509.928 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:509.994 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091509892 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:34:51:510.037 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:34:51:510.100 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:510.205 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739414091510075 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 821 in_bytes = 60679 out_pkts = 861 out_bytes = 59962 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 161 out_bw_usage = 132 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:34:51:510.355 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:34:51:510.371 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:510.385 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739414091510081 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:34:51:510.393 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:34:51:739.150 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:739.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091738973 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 368 dequeues = 368 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 429 dequeues = 429 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414063747140 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414091510368 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:34:51:739.259 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:34:51:739.274 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:739.772 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091739102 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 104 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 368, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:34:51:739.822 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:34:51:739.849 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:739.897 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091739697 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.335 five_min_avg = 0.39125 fifteen_min_avg = 0.4075 ] cpu_share = 0.270833 ] ] 2025-02-13 Thu 02:34:51:739.908 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:34:51:739.923 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:739.948 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091739709 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.270833 used_sys_mem = 4269468888 one_min_cpuload = 0.335 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:34:51:739.958 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:34:51:739.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:740.214 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091739742 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 821 in_bytes = 60679 out_tpkts = 861 out_bytes = 59962 exception_packets = 64 exception_packets_allowed = 49 raw_vhost_stats= [ name = vhost0 in_pkts = 821 in_bytes = 60679 out_pkts = 861 out_bytes = 59962 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 639 in_bytes = 50638 out_pkts = 830 out_bytes = 61016 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:34:51:740.241 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:34:51:740.255 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:34:51:740.302 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414091739770 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:34:51:740.313 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:34:53:747.285 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:34:53:747.517 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:34:53:747.627 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:53:747.657 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414093747370 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:34:53:747.677 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:34:53:747.686 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:34:53:747.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414093747614 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 106 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 374, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:34:53:747.996 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:21:510.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:510.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414121510674 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:35:21:510.871 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:35:21:510.885 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:510.995 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414121510801 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 945 in_bytes = 71721 out_pkts = 985 out_bytes = 71046 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 2944 out_bw_usage = 2955 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:35:21:511.056 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:35:21:511.074 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:511.092 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414121510806 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:35:21:511.102 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:35:21:739.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:739.585 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414121739276 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 379 dequeues = 379 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 441 dequeues = 441 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414093747377 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414121511071 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:35:21:739.604 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:21:739.616 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:739.897 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414121739377 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 107 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 379, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:35:21:739.930 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:21:739.941 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:740.108 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414121739914 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 945 in_bytes = 71721 out_tpkts = 985 out_bytes = 71046 exception_packets = 66 exception_packets_allowed = 51 raw_vhost_stats= [ name = vhost0 in_pkts = 945 in_bytes = 71721 out_pkts = 985 out_bytes = 71046 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 8.18573e-15 total_out_bandwidth_utilization = 8.41883e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 1510, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 1553, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 760 in_bytes = 61596 out_pkts = 954 out_bytes = 72058 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:35:21:740.124 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:35:21:740.133 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:21:740.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414121739947 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:35:21:740.171 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:35:23:747.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:35:23:747.769 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:35:23:747.810 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:35:23:747.914 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:35:23:748.043 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:23:748.073 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414123747847 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:35:23:748.081 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:35:23:748.090 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:35:23:748.128 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:23:748.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414123748001 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 109 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 383, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:35:23:748.426 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:51:510.128 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:510.192 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414151510089 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:35:51:510.206 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:35:51:510.329 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:510.437 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414151510303 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 953 in_bytes = 72333 out_pkts = 993 out_bytes = 71530 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 168 out_bw_usage = 133 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:35:51:510.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:35:51:510.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:510.474 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414151510311 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:35:51:510.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:35:51:739.728 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:739.829 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151739511 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 388 dequeues = 388 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 452 dequeues = 452 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414123747912 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414151510460 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:35:51:739.844 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:51:739.854 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:740.127 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151739676 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 110 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 388, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:35:51:740.156 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:35:51:740.178 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:740.210 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151740135 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.55375 five_min_avg = 0.4725 fifteen_min_avg = 0.43625 ] cpu_share = 0.277083 ] ] 2025-02-13 Thu 02:35:51:740.217 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:35:51:740.229 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:740.245 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151740176 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.277083 used_sys_mem = 4269468888 one_min_cpuload = 0.55375 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:35:51:740.252 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:35:51:740.262 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:740.420 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151740208 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 953 in_bytes = 72333 out_tpkts = 993 out_bytes = 71530 exception_packets = 67 exception_packets_dropped = 16 raw_vhost_stats= [ name = vhost0 in_pkts = 953 in_bytes = 72333 out_pkts = 993 out_bytes = 71530 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 765 in_bytes = 61954 out_pkts = 962 out_bytes = 72670 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:35:51:740.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:35:51:740.442 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:35:51:740.472 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414151740249 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:35:51:740.479 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:35:53:748.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:35:53:748.339 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:35:53:748.465 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:53:748.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414153748182 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:35:53:748.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:35:53:748.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:35:53:748.874 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414153748444 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 112 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 38 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 49, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 394, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:35:53:748.910 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:10:979.744 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:35456 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:36:11:027.127 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:35456 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:36:21:510.299 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:510.371 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414181510262 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:36:21:510.383 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:36:21:510.503 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:510.609 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414181510475 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 966 in_bytes = 73224 out_pkts = 1006 out_bytes = 72313 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 237 out_bw_usage = 208 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:36:21:510.642 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:36:21:510.655 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:510.667 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414181510482 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:36:21:510.675 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:36:21:739.993 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:740.133 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414181739869 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 399 dequeues = 399 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 464 dequeues = 464 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414153748192 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414181510653 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:36:21:740.149 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:21:740.158 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:740.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414181739963 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 113 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 51, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 399, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:36:21:740.460 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:21:740.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:740.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414181740374 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 966 in_bytes = 73224 out_tpkts = 1006 out_bytes = 72313 exception_packets = 68 exception_packets_allowed = 52 raw_vhost_stats= [ name = vhost0 in_pkts = 966 in_bytes = 73224 out_pkts = 1006 out_bytes = 72313 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 7.31836e-16 total_out_bandwidth_utilization = 1.0842e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 135, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 200, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 775 in_bytes = 62611 out_pkts = 975 out_bytes = 73561 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:36:21:740.663 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:36:21:740.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:21:740.720 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414181740422 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:36:21:740.727 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:36:23:748.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:36:23:748.569 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:36:23:748.579 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:36:23:748.695 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Idle 2025-02-13 Thu 02:36:23:748.795 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:23:748.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414183748616 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:36:23:748.833 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:36:23:748.842 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:36:23:748.873 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:23:749.134 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414183748781 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 115 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 51, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 403, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:36:23:749.171 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:51:510.454 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:510.513 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414211510414 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:36:51:510.524 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:36:51:510.605 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:510.708 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414211510576 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 976 in_bytes = 73920 out_pkts = 1016 out_bytes = 72901 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 185 out_bw_usage = 156 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:36:51:510.730 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:36:51:510.741 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:510.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414211510584 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:36:51:510.775 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:36:51:740.266 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.356 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740092 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 408 dequeues = 408 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 475 dequeues = 475 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414183748693 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414211510739 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:36:51:740.371 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:51:740.381 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.640 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740216 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 116 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 51, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 408, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:36:51:740.668 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:36:51:740.678 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.709 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740527 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.3825 five_min_avg = 0.43875 fifteen_min_avg = 0.4275 ] cpu_share = 0.26875 ] ] 2025-02-13 Thu 02:36:51:740.716 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:36:51:740.726 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740551 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.26875 used_sys_mem = 4269468888 one_min_cpuload = 0.3825 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:36:51:740.752 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:36:51:740.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740585 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 976 in_bytes = 73920 out_tpkts = 1016 out_bytes = 72901 exception_packets = 69 exception_packets_allowed = 53 raw_vhost_stats= [ name = vhost0 in_pkts = 976 in_bytes = 73920 out_pkts = 1016 out_bytes = 72901 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 782 in_bytes = 63073 out_pkts = 985 out_bytes = 74257 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:36:51:740.928 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:36:51:740.939 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:36:51:740.968 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: SANDESH: Send FAILED: 1739414211740611 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:36:51:740.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672695199488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:36:53:748.895 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:36:53:749.224 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:36:53:749.369 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:53:749.418 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414213749039 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:36:53:749.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:36:53:749.443 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:36:53:749.692 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414213749347 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 118 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 40 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 51, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 414, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:36:53:749.735 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:03:918.115 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:46864 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:37:03:963.042 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:46864 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:37:15:323.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53730 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:37:15:367.189 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672865302848, Pid 1180635]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8085::10.0.0.17:53730 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:37:21:512.353 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:518.746 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414241512287 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:37:21:518.808 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:37:21:518.830 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:518.948 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414241512553 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 993 in_bytes = 75144 out_pkts = 1033 out_bytes = 74017 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 326 out_bw_usage = 297 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:37:21:518.986 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:37:21:519.000 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:519.035 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414241512566 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:37:21:519.045 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:37:21:740.570 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:740.683 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414241740397 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 419 dequeues = 419 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 487 dequeues = 487 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414213749065 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414241518998 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:37:21:740.697 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:21:740.706 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:741.118 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414241740526 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 119 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 419, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:37:21:741.164 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:21:741.181 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:741.384 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414241740723 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 993 in_bytes = 75144 out_tpkts = 1033 out_bytes = 74017 exception_packets = 70 exception_packets_allowed = 54 raw_vhost_stats= [ name = vhost0 in_pkts = 993 in_bytes = 75144 out_pkts = 1033 out_bytes = 74017 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 1.04626e-15 total_out_bandwidth_utilization = 1.38778e-15 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 193, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 256, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 796 in_bytes = 64063 out_pkts = 1002 out_bytes = 75481 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:37:21:741.403 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:37:21:741.413 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:21:741.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414241740758 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:37:21:741.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:37:23:749.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:37:23:749.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:37:23:749.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:37:23:749.621 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Idle 2025-02-13 Thu 02:37:23:749.768 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:23:749.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414243749550 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:37:23:749.812 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:37:23:749.821 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:37:23:749.869 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:23:750.212 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414243749750 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 121 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 423, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:37:23:750.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:51:512.421 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:512.499 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: SANDESH: Send FAILED: 1739414271512381 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:37:51:512.511 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672703596288, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:37:51:512.558 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:512.665 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414271512533 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 1003 in_bytes = 75792 out_pkts = 1046 out_bytes = 74767 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 178 out_bw_usage = 206 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:37:51:512.681 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:37:51:512.694 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:512.725 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414271512540 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:37:51:512.733 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:37:51:740.946 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740651 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 428 dequeues = 428 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 498 dequeues = 498 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414243749619 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414271512692 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:37:51:741.107 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:51:741.117 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.153 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740881 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.3125 five_min_avg = 0.40625 fifteen_min_avg = 0.41625 ] cpu_share = 0.335417 ] ] 2025-02-13 Thu 02:37:51:741.160 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:37:51:741.168 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.188 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740895 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.335417 used_sys_mem = 4269468888 one_min_cpuload = 0.3125 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:37:51:741.194 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:37:51:741.202 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.442 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740895 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 122 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 428, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:37:51:741.475 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:37:51:741.484 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.626 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740927 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 1003 in_bytes = 75792 out_tpkts = 1046 out_bytes = 74767 exception_packets = 71 exception_packets_allowed = 55 raw_vhost_stats= [ name = vhost0 in_pkts = 1003 in_bytes = 75792 out_pkts = 1046 out_bytes = 74767 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 806 in_bytes = 64687 out_pkts = 1012 out_bytes = 76129 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:37:51:741.639 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:37:51:741.650 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:37:51:741.679 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: SANDESH: Send FAILED: 1739414271740957 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:37:51:741.686 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672686802688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:37:53:749.781 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:37:53:750.100 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:37:53:750.224 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:53:750.262 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414273749873 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:37:53:750.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:37:53:750.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:37:53:750.608 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414273750209 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 124 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 434, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:37:53:750.632 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:38:21:512.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:512.637 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414301512528 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:38:21:512.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:38:21:512.658 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:512.729 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414301512653 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 1009 in_bytes = 76224 out_pkts = 1052 out_bytes = 75091 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 115 out_bw_usage = 86 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:38:21:512.743 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:38:21:512.752 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:512.763 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414301512656 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:38:21:512.783 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:38:21:741.282 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:741.405 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414301741106 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 439 dequeues = 439 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 510 dequeues = 510 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414273749881 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414301512750 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:38:21:741.421 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:38:21:741.432 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:741.586 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414301741146 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 in_tpkts = 1009 in_bytes = 76224 out_tpkts = 1052 out_bytes = 75091 raw_vhost_stats= [ name = vhost0 in_pkts = 1009 in_bytes = 76224 out_pkts = 1052 out_bytes = 75091 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] total_in_bandwidth_utilization = 5.9089e-16 total_out_bandwidth_utilization = 7.80626e-16 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = ens3 _iter260->second = 109, ] ] phy_band_out_bps= [ [ _iter261->first = ens3 _iter261->second = 144, ] ] raw_phy_if_stats= [ [ _iter262->first = ens3 [ in_pkts = 809 in_bytes = 64885 out_pkts = 1018 out_bytes = 76561 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = ens3 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 0 ], ] ] ] 2025-02-13 Thu 02:38:21:741.613 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:38:21:741.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:741.652 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414301741180 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 raw_xmpp_stats= [ [ _iter306->first = 10.0.0.17 [ reconnects = 1 in_msgs = 10 out_msgs = 31 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 8 inet4_multicast = 0 evpn = 0 bridge = 6 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 3 inet4_multicast = 0 evpn = 2 bridge = 3 inet6_unicast = 3 ], ] ] ] 2025-02-13 Thu 02:38:21:741.659 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: VrouterControlStatsTrace 2025-02-13 Thu 02:38:21:741.670 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:38:21:741.951 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414301741216 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 125 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 439, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:38:21:741.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:38:23:750.266 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:38:23:750.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:38:23:750.354 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:38:23:750.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Idle 2025-02-13 Thu 02:38:23:750.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:23:750.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414303750389 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:5269, ] ] status = Up description = OpenConfirm ], [ type = XMPP name = dns-server:10.0.0.17 server_addrs= [ [ (*_iter6) = 10.0.0.17:53, ] ] status = Up description = OpenConfirm ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:38:23:750.600 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:38:23:750.609 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:38:23:750.638 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:23:750.904 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: SANDESH: Send FAILED: 1739414303750549 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 127 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 443, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:38:23:750.935 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672707794688, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:38:51:514.384 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:514.447 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: SANDESH: Send FAILED: 1739414331514344 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = default-domain:default-project:ip-fabric ingress_flow_count = 0 egress_flow_count = 0 ] 2025-02-13 Thu 02:38:51:514.458 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672691001088, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UveVirtualNetworkAgentTrace 2025-02-13 Thu 02:38:51:514.589 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:514.693 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739414331514534 [SYS_NOTICE]: UVEVMIStats: data= [ name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 vm_name = vm_uuid = virtual_network = default-domain:default-project:ip-fabric raw_if_stats= [ in_pkts = 1019 in_bytes = 76872 out_pkts = 1062 out_bytes = 75631 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] in_bw_usage = 172 out_bw_usage = 144 raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 0 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 1 ] policy_rules= [ [ ] ] ] 2025-02-13 Thu 02:38:51:514.710 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: UVEVMIStats 2025-02-13 Thu 02:38:51:514.720 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:514.732 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: SANDESH: Send FAILED: 1739414331514572 [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:cn-jenkins-deploy-platform-juju-os-473-1:vhost0 eps= [ [ ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2025-02-13 Thu 02:38:51:514.739 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672699397888, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: EndpointSecurityStats 2025-02-13 Thu 02:38:51:741.433 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:741.564 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414331741282 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 448 dequeues = 448 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 521 dequeues = 521 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414303750450 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414331514719 ] max_sm_queue_count = 7 ] 2025-02-13 Thu 02:38:51:741.580 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:38:51:741.591 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:741.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414331741354 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ num_cpu = 8 sys_mem_info= [ total = 25602048 used = 0 free = 497393198 buffers = 32520 cached = 25465888 node_type = ] meminfo= [ virt = 32520 peakvirt = 483359128 res = 32520 ] cpuload= [ one_min_avg = 0.26375 five_min_avg = 0.37375 fifteen_min_avg = 0.405 ] cpu_share = 0.275 ] ] 2025-02-13 Thu 02:38:51:741.636 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: VrouterStats 2025-02-13 Thu 02:38:51:741.645 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:741.661 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414331741379 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 cpu_info= [ [ [ mem_virt = 32520 cpu_share = 0.275 used_sys_mem = 4269468888 one_min_cpuload = 0.26375 mem_res = 32520 ], ] ] ] 2025-02-13 Thu 02:38:51:741.668 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Wrong state: Idle for event: EvSandeshSend message: ComputeCpuStateTrace 2025-02-13 Thu 02:38:51:741.675 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:38:51:741.931 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139672716191488, Pid 1180635]: SANDESH: Send FAILED: 1739414331741390 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 1180635 http_port = 8085 start_time = 1739413131729287 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 19 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = EndpointSecurityStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 128 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMIStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UVEVMITags [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVMInterfaceAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VnObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppSslHandShakeMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppUnconsumedEvent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 55, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 448, _iter59->first = sent _iter59->second = 0, ] ] ]