++ LOG_DIR=/var/log/contrail ++ export CONTAINER_LOG_DIR=/var/log/contrail/config-database-nodemgr ++ CONTAINER_LOG_DIR=/var/log/contrail/config-database-nodemgr ++ mkdir -p /var/log/contrail/config-database-nodemgr ++ log_file=/var/log/contrail/config-database-nodemgr/console.log ++ touch /var/log/contrail/config-database-nodemgr/console.log ++ chmod 600 /var/log/contrail/config-database-nodemgr/console.log ++ exec +++ tee -a /var/log/contrail/config-database-nodemgr/console.log +++ date ++ echo 'INFO: =================== Mon Oct 13 01:37:47 AM UTC 2025 ===================' INFO: =================== Mon Oct 13 01:37:47 AM UTC 2025 =================== ++ LOG_LOCAL=1 ++ source /functions.sh ++ source /contrail-functions.sh +++ get_default_ip ++++ get_default_nic ++++ get_gateway_nic_for_ip 1 ++++ command -v ip ++++ local ip=1 +++++ ip route get 1 +++++ grep -o 'dev.*' +++++ awk '{print $2}' ++++ local iface=ens3 ++++ [[ ens3 == \l\o ]] ++++ echo ens3 +++ local nic=ens3 +++ get_ip_for_nic ens3 +++ local nic=ens3 +++ get_cidr_for_nic ens3 +++ command -v ip +++ local nic=ens3 +++ ip addr show dev ens3 +++ cut -d / -f 1 +++ awk '{print $2}' +++ head -n 1 +++ grep 'inet ' ++ DEFAULT_LOCAL_IP=10.0.0.129 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ VXLAN_VN_ID_MODE=automatic ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ SERVICE_CORE_MASK= ++ DPDK_CTRL_THREAD_MASK= ++ HUGE_PAGES= ++ HUGE_PAGES_DIR=/dev/hugepages ++ HUGE_PAGES_1GB=0 ++ HUGE_PAGES_2MB=0 ++ HUGE_PAGES_1GB_DIR= ++ HUGE_PAGES_2MB_DIR= ++ [[ 0 != 0 ]] ++ [[ 0 != 0 ]] ++ DPDK_MEM_PER_SOCKET=1024 ++ DPDK_COMMAND_ADDITIONAL_ARGS= ++ NIC_OFFLOAD_ENABLE=False ++ DPDK_ENABLE_VLAN_FWRD=False ++ DIST_SNAT_PROTO_PORT_LIST= ++ CLOUD_ORCHESTRATOR=openstack ++ CLOUD_ADMIN_ROLE=admin ++ AAA_MODE=rbac ++ AUTH_MODE=noauth ++ AUTH_PARAMS= ++ SSL_ENABLE=True ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ SELFSIGNED_CERTS_WITH_IPS=True ++ CONTROLLER_NODES=10.0.0.129 ++ ANALYTICS_ALARM_ENABLE=False ++ ANALYTICS_SNMP_ENABLE=False ++ ANALYTICSDB_ENABLE=False ++ ANALYTICS_NODES=10.0.0.129 ++ ANALYTICSDB_NODES=10.0.0.129 ++ ANALYTICS_SNMP_NODES=10.0.0.129 ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ QUERYENGINE_INTROSPECT_PORT=8091 +++ get_server_list ANALYTICS ':8081 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8081 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:8081 ' +++ '[' -n '10.0.0.129:8081 ' ']' +++ echo 10.0.0.129:8081 ++ ANALYTICS_SERVERS=10.0.0.129:8081 +++ get_server_list ANALYTICSDB ':9042 ' +++ local server_typ=ANALYTICSDB_NODES +++ local 'port_with_delim=:9042 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:9042 ' +++ '[' -n '10.0.0.129:9042 ' ']' +++ echo 10.0.0.129:9042 ++ ANALYTICSDB_CQL_SERVERS=10.0.0.129:9042 ++ ANALYTICS_API_VIP= ++ ANALYTICS_ALARM_NODES=10.0.0.129 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ BGP_ASN=64512 ++ ENABLE_4BYTE_AS=false ++ APPLY_DEFAULTS=true ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 +++ get_server_list ANALYTICS ':8086 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:8086 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:8086 ' +++ '[' -n '10.0.0.129:8086 ' ']' +++ echo 10.0.0.129:8086 ++ COLLECTOR_SERVERS=10.0.0.129:8086 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9041 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7201 ++ CONFIGDB_CASSANDRA_DRIVER=cql ++ CONFIG_NODES=10.0.0.129 ++ CONFIGDB_NODES=10.0.0.129 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ CONFIG_API_ADMIN_PORT=8095 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:8082 ' +++ '[' -n '10.0.0.129:8082 ' ']' +++ echo 10.0.0.129:8082 ++ CONFIG_SERVERS=10.0.0.129:8082 +++ get_server_list CONFIGDB ':9161 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9161 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:9161 ' +++ '[' -n '10.0.0.129:9161 ' ']' +++ echo 10.0.0.129:9161 ++ CONFIGDB_SERVERS=10.0.0.129:9161 +++ get_server_list CONFIGDB ':9041 ' +++ local server_typ=CONFIGDB_NODES +++ local 'port_with_delim=:9041 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:9041 ' +++ '[' -n '10.0.0.129:9041 ' ']' +++ echo 10.0.0.129:9041 ++ CONFIGDB_CQL_SERVERS=10.0.0.129:9041 ++ CONFIG_API_VIP= ++ CONFIG_API_SSL_ENABLE=True ++ CONFIG_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CONFIG_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CONFIG_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CONFIG_API_WORKER_COUNT=1 ++ CONFIG_API_MAX_REQUESTS=1024 ++ ANALYTICS_API_SSL_ENABLE=True ++ ANALYTICS_API_SSL_INSECURE=True ++ ANALYTICS_API_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ ANALYTICS_API_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ ANALYTICS_API_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_ENABLE=True ++ CASSANDRA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ CASSANDRA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ CASSANDRA_SSL_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ CASSANDRA_SSL_KEYSTORE_PASSWORD=astrophytum ++ CASSANDRA_SSL_TRUSTSTORE_PASSWORD=ornatum ++ CASSANDRA_SSL_PROTOCOL=TLS ++ CASSANDRA_SSL_ALGORITHM=SunX509 ++ CASSANDRA_SSL_CIPHER_SUITES='[TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]' ++ CASSANDRA_CONFIG_MEMTABLE_FLUSH_WRITER=4 ++ CASSANDRA_CONFIG_CONCURRECT_COMPACTORS=4 ++ CASSANDRA_CONFIG_COMPACTION_THROUGHPUT_MB_PER_SEC=256 ++ CASSANDRA_CONFIG_CONCURRECT_READS=64 ++ CASSANDRA_CONFIG_CONCURRECT_WRITES=64 ++ CASSANDRA_CONFIG_MEMTABLE_ALLOCATION_TYPE=offheap_objects ++ CASSANDRA_REAPER_ENABLED=false ++ CASSANDRA_REAPER_JMX_KEY=reaperJmxKey ++ CASSANDRA_REAPER_JMX_AUTH_USERNAME=reaperUser ++ CASSANDRA_REAPER_JMX_AUTH_PASSWORD=reaperPass ++ CASSANDRA_REAPER_APP_PORT=8071 ++ CASSANDRA_REAPER_ADM_PORT=8072 ++ CONTROL_NODES=10.0.0.129 ++ CONTROL_INTROSPECT_PORT=8083 ++ DNS_NODES=10.0.0.129 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ USE_EXTERNAL_TFTP=False ++ ZOOKEEPER_NODES=10.0.0.129 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 +++ get_server_list ZOOKEEPER :2181, +++ local server_typ=ZOOKEEPER_NODES +++ local port_with_delim=:2181, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+=10.0.0.129:2181, +++ '[' -n 10.0.0.129:2181, ']' +++ echo 10.0.0.129:2181 ++ ZOOKEEPER_SERVERS=10.0.0.129:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:2181 ' +++ '[' -n '10.0.0.129:2181 ' ']' +++ echo 10.0.0.129:2181 ++ ZOOKEEPER_SERVERS_SPACE_DELIM=10.0.0.129:2181 ++ RABBITMQ_NODES=10.0.0.129 ++ RABBITMQ_NODE_PORT=5673 +++ get_server_list RABBITMQ :5673, +++ local server_typ=RABBITMQ_NODES +++ local port_with_delim=:5673, +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+=10.0.0.129:5673, +++ '[' -n 10.0.0.129:5673, ']' +++ echo 10.0.0.129:5673 ++ RABBITMQ_SERVERS=10.0.0.129:5673 ++ RABBITMQ_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=true ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=True ++ RABBITMQ_SSL_VER=tlsv1.2 ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ RABBITMQ_HEARTBEAT_INTERVAL=60 ++ RABBITMQ_CLUSTER_PARTITION_HANDLING=autoheal ++ RABBITMQ_MIRRORED_QUEUE_MODE=all ++ REDIS_SERVER_PORT=6379 ++ REDIS_SERVER_PASSWORD= +++ get_server_list ANALYTICS ':6379 ' +++ local server_typ=ANALYTICS_NODES +++ local 'port_with_delim=:6379 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:6379 ' +++ '[' -n '10.0.0.129:6379 ' ']' +++ echo 10.0.0.129:6379 ++ REDIS_SERVERS=10.0.0.129:6379 ++ REDIS_LISTEN_ADDRESS= ++ REDIS_PROTECTED_MODE= ++ REDIS_SSL_ENABLE=True ++ REDIS_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ REDIS_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ REDIS_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' redis_ssl_config ++ true ++ KAFKA_NODES=10.0.0.129 ++ KAFKA_PORT=9092 +++ get_server_list KAFKA ':9092 ' +++ local server_typ=KAFKA_NODES +++ local 'port_with_delim=:9092 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in "${server_list[@]}" ++++ echo 10.0.0.129 +++ local server_address=10.0.0.129 +++ extended_server_list+='10.0.0.129:9092 ' +++ '[' -n '10.0.0.129:9092 ' ']' +++ echo 10.0.0.129:9092 ++ KAFKA_SERVERS=10.0.0.129:9092 ++ KAFKA_SSL_ENABLE=True ++ KAFKA_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ KAFKA_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ KAFKA_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=password ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=admin_domain ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=admin_domain ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=252.129.144.224 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=5050 ++ KEYSTONE_AUTH_PUBLIC_PORT=5050 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ KEYSTONE_AUTH_ENDPOINT_TYPE= ++ KEYSTONE_AUTH_SYNC_ON_DEMAND= ++ KEYSTONE_AUTH_INTERFACE=public ++ KUBEMANAGER_NODES=10.0.0.129 ++ KUBERNETES_CLUSTER_NAME=k8s ++ KUBERNETES_CNI_META_PLUGIN=multus ++ METADATA_PROXY_SECRET=contrail ++ BARBICAN_TENANT_NAME=service ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=password ++ AGENT_MODE=kernel ++ EXTERNAL_ROUTERS= ++ SUBCLUSTER= ++ VROUTER_COMPUTE_NODE_ADDRESS= ++ VROUTER_MODULE_OPTIONS= ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ TSN_EVPN_MODE=False ++ TSN_NODES='[]' ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True ++ SLO_DESTINATION=collector ++ '[' -n '' ']' ++ SAMPLE_DESTINATION=collector ++ FLOW_EXPORT_RATE=0 ++ WEBUI_NODES=10.0.0.129 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/contrail/webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/contrail/webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ WEBUI_STATIC_AUTH_USER=admin ++ WEBUI_STATIC_AUTH_PASSWORD=contrail123 ++ WEBUI_STATIC_AUTH_ROLE=cloudAdmin ++ XMPP_SERVER_PORT=5269 ++ XMPP_SSL_ENABLE=True ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ INTROSPECT_SSL_ENABLE=True ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_LISTEN_ALL=True ++ SANDESH_SSL_ENABLE=True ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ CONFIGURE_IPTABLES=false ++ FWAAS_ENABLE=False ++ CONTAINERD_NAMESPACE=k8s.io ++ TOR_AGENT_OVS_KA=10000 ++ TOR_TYPE=ovs ++ TOR_OVS_PROTOCOL=tcp ++ TORAGENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ TORAGENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ TORAGENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ [[ /v3 == \/\v\2\.\0 ]] ++ [[ openstack == \o\p\e\n\s\t\a\c\k ]] ++ AUTH_MODE=keystone ++ [[ keystone == \k\e\y\s\t\o\n\e ]] ++ AUTH_PARAMS='--admin_password password' ++ AUTH_PARAMS+=' --admin_tenant_name admin' ++ AUTH_PARAMS+=' --admin_user admin' ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' xmpp_certs_config ++ true ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' analytics_api_ssl_opts ++ true ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' rabbitmq_ssl_config ++ true ++ read -r -d '' kombu_ssl_config ++ true ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ read -r -d '' kafka_ssl_config ++ true ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z '' ]] ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ export TSN_AGENT_MODE= ++ TSN_AGENT_MODE= ++ [[ -n '' ]] ++ collector_stats_config= ++ [[ -z x ]] ++ RSYSLOGD_XFLOW_LISTEN_PORT=9898 + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + wait_files /etc/contrail/ssl/certs/server.pem /etc/contrail/ssl/private/server-privkey.pem + local file1=/etc/contrail/ssl/certs/server.pem + local file2=/etc/contrail/ssl/private/server-privkey.pem + printf 'INFO: wait for files /etc/contrail/ssl/certs/server.pem and /etc/contrail/ssl/private/server-privkey.pem' INFO: wait for files /etc/contrail/ssl/certs/server.pem and /etc/contrail/ssl/private/server-privkey.pem+ wait_cmd_success 'test -f /etc/contrail/ssl/certs/server.pem' + local 'cmd=test -f /etc/contrail/ssl/certs/server.pem' + local interval=3 + local max=20 + local i=0 + test -f /etc/contrail/ssl/certs/server.pem + return 0 + wait_cmd_success 'test -f /etc/contrail/ssl/private/server-privkey.pem' + local 'cmd=test -f /etc/contrail/ssl/private/server-privkey.pem' + local interval=3 + local max=20 + local i=0 + test -f /etc/contrail/ssl/private/server-privkey.pem + return 0 + echo -e '\nINFO: files /etc/contrail/ssl/certs/server.pem and /etc/contrail/ssl/private/server-privkey.pem are present' INFO: files /etc/contrail/ssl/certs/server.pem and /etc/contrail/ssl/private/server-privkey.pem are present + [[ /etc/contrail/ssl/private/server-privkey.pem != \/\e\t\c\/\c\o\n\t\r\a\i\l\/\s\s\l\/\p\r\i\v\a\t\e\/\s\e\r\v\e\r\-\p\r\i\v\k\e\y\.\p\e\m ]] + [[ /etc/contrail/ssl/private/server-privkey.pem != \/\e\t\c\/\c\o\n\t\r\a\i\l\/\s\s\l\/\p\r\i\v\a\t\e\/\s\e\r\v\e\r\-\p\r\i\v\k\e\y\.\p\e\m ]] + [[ /etc/contrail/ssl/private/server-privkey.pem != \/\e\t\c\/\c\o\n\t\r\a\i\l\/\s\s\l\/\p\r\i\v\a\t\e\/\s\e\r\v\e\r\-\p\r\i\v\k\e\y\.\p\e\m ]] + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + cat + [[ keystone == \k\e\y\s\t\o\n\e ]] + cat + [[ http == \h\t\t\p\s ]] + mv -f /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + [[ config-database == \t\o\r\a\g\e\n\t ]] + export NODEMGR_TYPE=contrail-config-database + NODEMGR_TYPE=contrail-config-database + NODEMGR_NAME=contrail-config-database-nodemgr ++ echo CONFIG-DATABASE ++ tr - _ + ntype=CONFIG_DATABASE + [[ CONFIG_DATABASE == \V\R\O\U\T\E\R ]] + [[ CONFIG_DATABASE == \D\A\T\A\B\A\S\E ]] + [[ CONFIG_DATABASE == \C\O\N\F\I\G\_\D\A\T\A\B\A\S\E ]] + htype=CONFIGDB + [[ CONFIG_DATABASE == \C\O\N\T\R\O\L ]] ++ get_listen_ip_for_node CONFIGDB +++ find_my_ip_and_order_for_node CONFIGDB +++ local server_typ=CONFIGDB_NODES +++ find_my_ip_and_order_for_node_list 10.0.0.129 +++ local servers=10.0.0.129 +++ local server_list= +++ IFS=, +++ read -ra server_list ++++ tr '\n' , ++++ get_local_ips ++++ awk '/32 host/ { print f } {f=$2}' ++++ cat /proc/net/fib_trie ++++ grep -vi host +++ cut -d ' ' -f 1 ++++ sort ++++ uniq +++ local local_ips=,10.0.0.129,10.0.215.1,127.0.0.1,172.17.0.1,252.129.0.1,, +++ local ord=1 +++ for server in "${server_list[@]}" +++ local ret=0 ++++ python3 -c 'import socket; print(socket.gethostbyname('\''10.0.0.129'\''))' +++ local server_ip=10.0.0.129 +++ [[ 0 == 0 ]] +++ [[ -n 10.0.0.129 ]] +++ [[ ,10.0.0.129,10.0.215.1,127.0.0.1,172.17.0.1,252.129.0.1,, =~ ,10\.0\.0\.129, ]] +++ echo 10.0.0.129 1 +++ return ++ local ip=10.0.0.129 ++ [[ -z 10.0.0.129 ]] ++ echo 10.0.0.129 + hostip=10.0.0.129 + introspect_ip=0.0.0.0 + is_enabled True + local val=true + [[ true == \t\r\u\e ]] + mkdir -p /etc/contrail + cat ++ format_boolean True ++ is_enabled True ++ local val=true ++ [[ true == \t\r\u\e ]] ++ echo True + '[' -n '' ']' + cat + add_ini_params_from_env CONFIG_DATABASE_NODEMGR /etc/contrail/contrail-config-database-nodemgr.conf + local service_name=CONFIG_DATABASE_NODEMGR + local cfg_path=/etc/contrail/contrail-config-database-nodemgr.conf + local delim=__ ++ set -o posix ++ set ++ cut -d = -f 1 ++ sed 's/^CONFIG_DATABASE_NODEMGR__//g' ++ grep '^CONFIG_DATABASE_NODEMGR__.*__.*=.*$' ++ sort + local vars=DEFAULTS__minimum_diskGB + local section= + for var in $vars + local var_name=CONFIG_DATABASE_NODEMGR__DEFAULTS__minimum_diskGB + local val=4 ++ echo DEFAULTS__minimum_diskGB ++ sed s/DASH/-/g + local var_unsubst=DEFAULTS__minimum_diskGB ++ sed 's/^\(.*\)__.*$/\1/' ++ echo DEFAULTS__minimum_diskGB + local var_section=DEFAULTS + [[ '' != \D\E\F\A\U\L\T\S ]] + echo '[DEFAULTS]' + section=DEFAULTS ++ sed 's/.*__\(.*\)$/\1/' ++ echo DEFAULTS__minimum_diskGB + local var_param=minimum_diskGB + echo 'minimum_diskGB = 4' + cat /etc/contrail/contrail-config-database-nodemgr.conf [DEFAULTS] http_server_ip=0.0.0.0 log_file=/var/log/contrail/config-database-nodemgr/contrail-config-database-nodemgr.log log_level=SYS_DEBUG log_local=1 hostip=10.0.0.129 db_port=9041 db_jmx_port=7201 db_use_ssl=True [COLLECTOR] server_list=10.0.0.129:8086 [SANDESH] introspect_ssl_enable=True introspect_ssl_insecure=True sandesh_ssl_enable=True sandesh_keyfile=/etc/contrail/ssl/private/server-privkey.pem sandesh_certfile=/etc/contrail/ssl/certs/server.pem sandesh_server_keyfile=/etc/contrail/ssl/private/server-privkey.pem sandesh_server_certfile=/etc/contrail/ssl/certs/server.pem sandesh_ca_cert=/etc/contrail/ssl/certs/ca-cert.pem [DEFAULTS] minimum_diskGB = 4 + upgrade_old_logs contrail-config-database-nodemgr + local template=contrail-config-database-nodemgr ++ dirname /var/log/contrail/config-database-nodemgr + local old_dir=/var/log/contrail + mkdir -p /var/log/contrail/config-database-nodemgr + chmod 755 /var/log/contrail/config-database-nodemgr + mv -n '/var/log/contrail/*contrail-config-database-nodemgr.log*' /var/log/contrail/config-database-nodemgr/ + true + [[ -n 1999 ]] + [[ -n 1999 ]] + local owner_opts=1999:1999 + chown 1999:1999 /var/log/contrail/config-database-nodemgr + find /var/log/contrail/config-database-nodemgr -uid 0 -exec chown 1999:1999 '{}' + + exec /bin/sh -c '/usr/local/bin/contrail-nodemgr --nodetype=${NODEMGR_TYPE}' 10/13/2025 01:37:50.082 7fa205e5a240 [contrail-config-database-nodemgr] [INFO]: SANDESH: CONNECT TO COLLECTOR: True 10/13/2025 01:37:50.354 7fa205e5a240 [contrail-config-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 10/13/2025 01:37:50.384 7fa205e5a240 [contrail-config-database-nodemgr] [INFO]: SANDESH: INTROSPECT IS ON: 0.0.0.0:8112 10/13/2025 01:37:50.402 7fa205e5a240 [contrail-config-database-nodemgr] [ERROR]: Failed to import package "database.sandesh" 10/13/2025 01:37:50.408 7fa205e5a240 [contrail-config-database-nodemgr] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 10/13/2025 01:37:50.409 7fa205e5a240 [contrail-config-database-nodemgr] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/config-database-nodemgr/contrail-config-database-nodemgr.log] CRITICAL:root:Exit data: {'ID': '558403436a62ccf5b3925224895218f6735b79f80d221dc33ac8e31abbd93a29', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1072850} ERROR:contrail-config-database-nodemgr:Failed to connect to database by CQL and socket: Result: 1 INFO:contrail-config-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-os-589-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1760319472954252, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-config-database-nodemgr', instance_id='0', state='Non-Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=[''], status='Down', description='none to Idle on EvStart')]), flag_infos=None, description='Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Cassandra state detected DOWN. Collector:Collector[none to Idle on EvStart] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:37:54] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 4597 0.002025 CRITICAL:root:Exit data: {'ID': '6ae14db912037fd8d92a7f1c37796ee78f4bb32647fcd79fcfd71ed000595232', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1073104} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 CRITICAL:root:Exit data: {'ID': '393ebee94e7604c1d6195c5fb7da2614bd60a5ccb8b5c0c55382ed6fa58fe207', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'config_db_uuid'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1073871} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr config_db_uuid'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': '3e8c132efe8e1b85e0332485a6ad9bfb085a8ec7e3ef05246cedf98ee799e8f3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'to_bgp_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1074086} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr to_bgp_keyspace'. Error is: Result: 1 ERROR:contrail-config-database-nodemgr:Session Event: TCP Connect Fail INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Cassandra state detected DOWN. Collector:Collector[Connect to Idle on EvTcpConnectFail] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-os-589-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8112 start_time = 1760319470434168 collector_name = collector_ip = 10.0.0.129:8086 collector_list = [ 10.0.0.129:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] CRITICAL:root:Exit data: {'ID': '4d3e08b1ec2840a8709b97262afd1885783b55ff81890b02602f91b2289f36d9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'svc_monitor_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1074269} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr svc_monitor_keyspace'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': '994ebbc576a9488240fff02394cc19588750906a16c7910d194ce253052ecc02', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'DISCOVERY_SERVER'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1074432} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr DISCOVERY_SERVER'. Error is: Result: 1 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] CRITICAL:root:Exit data: {'ID': '82792f2f53fa382fb49c1e062d4a4e4ff51d570190d4a2cc1eceb616c3441479', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'useragent'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1074946} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr useragent'. Error is: Result: 1 CRITICAL:root:Exit data: {'ID': '1e0bfe1e3c3be6cf7942eaf6983ffb8924f206c6f870a131e45962d2fb55989d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'repair', '-pr', 'dm_keyspace'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1075587} ERROR:contrail-config-database-nodemgr:Failed to run cmd: 'nodetool -p 7201 repair -pr dm_keyspace'. Error is: Result: 1 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:38:05] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 4597 0.002257 INFO:contrail-config-database-nodemgr:Session Event: TCP Connected INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-config-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connection Closed INFO:contrail-config-database-nodemgr:Processing event[EvTcpClose] in state[ClientInit] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpClose] => State[ClientInit] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Down description = ClientInit to Idle on EvTcpClose >>, ] description = Cassandra state detected DOWN. Collector:Collector[ClientInit to Idle on EvTcpClose] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-os-589-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 1 pid = 1 http_port = 8112 start_time = 1760319470434168 collector_name = collector_ip = 10.0.0.129:8086 collector_list = [ 10.0.0.129:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 2 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-config-database-nodemgr:send_process_state_db: Sending UVE: NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-os-589-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1760319489714860, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=False, status=None, process_status=None, process_info=[ProcessInfo(process_name='cassandra', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1760319467000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='zookeeper', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1760319467000000', last_stop_time='', last_exit_time='', core_file_list=[]), ProcessInfo(process_name='contrail-config-database-nodemgr', process_state='PROCESS_STATE_RUNNING', start_count=1, stop_count=0, exit_count=0, last_start_time='1760319467000000', last_stop_time='', last_exit_time='', core_file_list=[])], disk_usage_info=None, description=None, all_core_file_list=None, build_info='{"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-10-13 00:48:09.442511", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-11624-1.novalocal", "build-id" : "package contrail", "build-number" : "lib is not installed\n"}]}', process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54258876, partition_space_available_1k=26831608, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=[], build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1297840, cpu_share=0.0, mem_res=1258796, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=172080, cpu_share=0.0, mem_res=117228, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=52148, cpu_share=0.0, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=14722668, free=10385176, buffers=752784, cached=7319096, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=8.58, five_min_avg=5.0, fifteen_min_avg=2.98, cpu_share=0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') INFO:contrail-config-database-nodemgr:Session Event: TCP Connected INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 deleted = False process_info = [ << process_name = cassandra process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1760319467000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = zookeeper process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1760319467000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, << process_name = contrail-config-database-nodemgr process_state = PROCESS_STATE_RUNNING start_count = 1 stop_count = 0 exit_count = 0 last_start_time = 1760319467000000 last_stop_time = last_exit_time = core_file_list = [ ] >>, ] build_info = {"build-info" : [{"build-version" : "v0+nightly", "build-time" : "2025-10-13 00:48:09.442511", "build-user" : "root", "build-hostname" : "jenkins-build-prepare-sandbox-11624-1.novalocal", "build-id" : "package contrail", "build-number" : "lib is not installed "}]} >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 disk_usage_info = { /dev/vda1 : << partition_type = ext4 partition_space_used_1k = 54258876 partition_space_available_1k = 26831608 percentage_partition_space_used = 67 >> } all_core_file_list = [ ] process_mem_cpu_usage = { cassandra : << mem_virt = 1297840 cpu_share = 0.0 mem_res = 1258796 >>zookeeper : << mem_virt = 172080 cpu_share = 0.0 mem_res = 117228 >>contrail-config-database-nodemgr : << mem_virt = 52148 cpu_share = 0.0 mem_res = 42704 >> } system_mem_usage = << total = 32665120 used = 14722668 free = 10385176 buffers = 752784 cached = 7319096 node_type = config-database-node >> system_cpu_usage = << one_min_avg = 8.58 five_min_avg = 5.0 fifteen_min_avg = 2.98 cpu_share = 0 node_type = config-database-node >> >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-config-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] ERROR:contrail-config-database-nodemgr:Session Event: TCP Connection Closed INFO:contrail-config-database-nodemgr:Processing event[EvTcpClose] in state[ClientInit] INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpClose] => State[ClientInit] -> State[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Down description = ClientInit to Idle on EvTcpClose >>, ] description = Cassandra state detected DOWN. Collector:Collector[ClientInit to Idle on EvTcpClose] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = cn-jenkins-deploy-platform-juju-os-589-1:ConfigDatabase:contrail-config-database-nodemgr:0 client_info = << status = Idle successful_connections = 2 pid = 1 http_port = 8112 start_time = 1760319470434168 collector_name = collector_ip = 10.0.0.129:8086 collector_list = [ 10.0.0.129:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 4 >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:38:12] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6903 0.001259 INFO:contrail-config-database-nodemgr:Processing event[EvIdleHoldTimerExpired] in state[Idle] INFO:contrail-config-database-nodemgr:Session Event: TCP Connected INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-config-database-nodemgr:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = cn-jenkins-deploy-platform-juju-os-589-1 process_status = [ << module_id = contrail-config-database-nodemgr instance_id = 0 state = Non-Functional connection_infos = [ << type = Collector name = Collector server_addrs = [ 10.0.0.129:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Cassandra state detected DOWN. Collector:Collector[Idle to Connect on EvIdleHoldTimerExpired] connection down >>, ] >> INFO:contrail-config-database-nodemgr:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-config-database-nodemgr:Processing event[EvTcpConnected] in state[Connect] DEBUG:contrail-config-database-nodemgr:Send sandesh control message. uve type count # 6 INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvTcpConnected] => State[Connect] -> State[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshUVESend] in state[ClientInit] DEBUG:contrail-config-database-nodemgr:Extracted sandesh length: 0000001012 DEBUG:contrail-config-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[ClientInit] DEBUG:contrail-config-database-nodemgr:Number of uve types in sandesh control message is 0 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:DatabaseUsage] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-config-database-nodemgr:Sandesh Client: Event[EvSandeshCtrlMessageRecv] => State[ClientInit] -> State[Established] INFO:contrail-config-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-config-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-config-database-nodemgr:Extracted sandesh length: 0000001432 DEBUG:contrail-config-database-nodemgr:Received sandesh control message [SandeshCtrlServerToClient] INFO:contrail-config-database-nodemgr:Processing event[EvSandeshCtrlMessageRecv] in state[Established] DEBUG:contrail-config-database-nodemgr:Number of uve types in sandesh control message is 3 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:DatabaseUsage] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-config-database-nodemgr:Sandesh Send Level [INVALID] -> [SYS_DEBUG] INFO:contrail-config-database-nodemgr:Sandesh Send Level [SYS_DEBUG] -> [INVALID] DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:38:25] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6853 0.001409 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:38:43] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6853 0.001422 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c43241258e3c9f068b2de5a58cf5c94f259c9275e5dd1fbed320e7010f44edeb', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1087678} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open INFO:contrail-config-database-nodemgr:send_nodemgr_process_status: Sending UVE:NodeStatusUVE(_context='', _scope='', _module='contrail-config-database-nodemgr', _source='cn-jenkins-deploy-platform-juju-os-589-1', _node_type='ConfigDatabase', _instance_id='0', _timestamp=1760319531370728, _versionsig=3672106175, _type=6, _hints=1, _client_context='', _client=None, _role=0, _logger=None, _level=2147483647, _category='', _send_queue_enabled=True, _http_server=None, _connect_to_collector=True, _disable_sending_object_logs=False, _disable_sending_all_messages=False, _more=False, data=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=[ProcessStatus(module_id='contrail-config-database-nodemgr', instance_id='0', state='Functional', connection_infos=dict_values([ConnectionInfo(type='Collector', name='Collector', server_addrs=['10.0.0.129:8086'], status='Up', description='ClientInit to Established on EvSandeshCtrlMessageRecv')]), flag_infos=None, description='')], process_info=None, disk_usage_info=None, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage=None, system_cpu_info=None, system_mem_usage=None, system_cpu_usage=None, running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo'), _seqnum=0) CRITICAL:root:Exit data: {'ID': '030f67defbcb87275cb58148c610b5b27211bfe44d938f9c7d6626d45453d4f6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1087699} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54298188, partition_space_available_1k=26792296, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1342308, cpu_share=0.54, mem_res=1329044, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=188124, cpu_share=0.03, mem_res=143296, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44764, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16752548, free=8542052, buffers=772772, cached=7065576, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=7.21, five_min_avg=5.34, fifteen_min_avg=3.21, cpu_share=6.12, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:39:10] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6824 0.002001 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '7f061914d7cceaea76024a1c42a3938cb417346102e022d2f518f37e88844de4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1090726} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2b5d6ab6a24262b9e9080cd46d7fc2e8fa669a092d4b00936b4e102ea84fd1c9', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1090746} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54307792, partition_space_available_1k=26782692, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1338960, cpu_share=0.04, mem_res=1326548, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=188552, cpu_share=0.0, mem_res=143716, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44776, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16557916, free=8714128, buffers=773104, cached=7088000, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.63, five_min_avg=4.65, fifteen_min_avg=3.11, cpu_share=2.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'b613e4f98f68b4aebc1793f434115a2f47c340410a4a284e43575b2ef77e9a8b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1105997} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c7b0e163dfabe9be3ee2184b8993e288e2aeb5c302a91ec44840b6cee1b604dd', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1106135} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54319836, partition_space_available_1k=26770648, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1341744, cpu_share=0.04, mem_res=1329328, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=190216, cpu_share=0.01, mem_res=145372, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44776, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16626832, free=7166056, buffers=809120, cached=8531060, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=7.8, five_min_avg=5.43, fifteen_min_avg=3.46, cpu_share=5.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:41:05] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001443 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:41:09] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001422 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:41:15] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001409 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c53d63333919bfc86803197cfd825f9e1e918e7b9e1155933e7a97ba0788e37e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1115223} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '9511ad75997d9d3bf6e385d7037e30bfb4aa2c2955169197b69467ba7ddaea66', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1115244} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54234880, partition_space_available_1k=26855604, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1341760, cpu_share=0.03, mem_res=1329340, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=190692, cpu_share=0.0, mem_res=145848, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44784, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16533428, free=7242128, buffers=809336, cached=8548076, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.16, five_min_avg=4.83, fifteen_min_avg=3.37, cpu_share=2.88, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:42:07] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6823 0.001667 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '87b3aae2c3e31d45fc4bf10e04e322762a37b51091f3e2010ffd90b3cb444a08', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1118729} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b712e0493ded312b8ed700759be08153ba9e5dabb150f07a6685739d1f2f2bed', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1118769} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54254640, partition_space_available_1k=26835844, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1341780, cpu_share=0.03, mem_res=1329352, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=191152, cpu_share=0.0, mem_res=146308, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44812, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16658960, free=6968488, buffers=809420, cached=8696096, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.41, five_min_avg=4.51, fifteen_min_avg=3.35, cpu_share=4.0, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '426ed4e8e601fae112e43b3ae256e839e6afe82f0544b657ffd40ec4cf1c244b', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1120613} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'beab61c3f364411675f60d9794871f5ce27324e03622cb4b1a2e74067c593adc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1120634} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54248184, partition_space_available_1k=26842300, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343504, cpu_share=0.04, mem_res=1330456, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=191600, cpu_share=0.0, mem_res=146752, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44820, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16630396, free=6975480, buffers=809424, cached=8716700, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.45, five_min_avg=4.02, fifteen_min_avg=3.26, cpu_share=2.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '1454a1c35fe68096d62b07b80e9c2d4d91b5eceedf9c485820bac14fa914b6b0', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1121429} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b398e9e83aa58a99d28932a120dec7c8bd67a548d26c72aaad1db257e7f2d542', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1121450} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54254876, partition_space_available_1k=26835608, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343576, cpu_share=0.03, mem_res=1330528, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=192068, cpu_share=0.0, mem_res=147220, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44820, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16635544, free=6962580, buffers=809424, cached=8724436, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=2.22, five_min_avg=3.66, fifteen_min_avg=3.18, cpu_share=2.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:45:44] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001551 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:45:49] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001304 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0f3b516942db01443b0a9bc704afd3fe10c4f27a1228dbad865b267ba76dde56', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1143125} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5d037fe09c558c40c451e214da3f209425155fe1e60c4ac1c48ff3e28324f33c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1143291} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:45:54] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6822 0.001883 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54271152, partition_space_available_1k=26819332, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343664, cpu_share=0.04, mem_res=1330616, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=194456, cpu_share=0.0, mem_res=149576, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44828, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16693632, free=6873720, buffers=809444, cached=8755204, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.39, five_min_avg=4.34, fifteen_min_avg=3.45, cpu_share=4.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '0cc89b34a689dedeb76b1c92aa97129f96d095d492418c7530f024154ebe89c3', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1145825} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '21f7e6172fa8c0698f8533f547203bf5beb466aec178f8e09664eb70a629919d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1145846} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:46:56] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6823 0.001719 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=54280288, partition_space_available_1k=26810196, percentage_partition_space_used=67, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343704, cpu_share=0.03, mem_res=1330652, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=194872, cpu_share=0.0, mem_res=149996, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44836, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16772928, free=6780304, buffers=809452, cached=8769184, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.42, five_min_avg=3.96, fifteen_min_avg=3.37, cpu_share=2.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '58fc4c58f69c11862ba6d29ca0fc58441ca67caa138d0bbbb4d14d0449c9a843', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1148247} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '1349bcd45332ba97fa3c5f58a5b83fdd62038d74fff36c1f30c1990c34411b59', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1148268} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55497520, partition_space_available_1k=25592964, percentage_partition_space_used=68, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1343692, cpu_share=0.03, mem_res=1330664, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=195344, cpu_share=0.0, mem_res=150472, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44836, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=16934908, free=5320304, buffers=840584, cached=10035900, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.71, five_min_avg=3.89, fifteen_min_avg=3.38, cpu_share=3.38, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'd469bb6ec64514603a8c43b7008d1ae271a7b00c5964d9d4a6cc4bc91083cf55', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1149522} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '6420178447c8790175ed3f5e9514fb9d1dc134e76bddff21bfba8ad6650d165c', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1149543} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55508552, partition_space_available_1k=25581932, percentage_partition_space_used=68, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1345356, cpu_share=0.07, mem_res=1331680, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=196244, cpu_share=0.0, mem_res=151364, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44840, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=17708304, free=4470932, buffers=844152, cached=10107220, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=3.44, five_min_avg=3.78, fifteen_min_avg=3.37, cpu_share=3.88, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8fa1049cb2a03cefe0964378141707f92a7a95eac6d221a34da66eee74916fe8', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1166803} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'b13b8ffbc0ca79fe3e992200f56bf6d216699ab129b06da7bac49fc90f9120d2', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1166882} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55564720, partition_space_available_1k=25525764, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1347692, cpu_share=0.13, mem_res=1333956, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=198860, cpu_share=0.0, mem_res=153900, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44848, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18646808, free=3443284, buffers=847484, cached=10192988, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=11.27, five_min_avg=5.7, fifteen_min_avg=4.04, cpu_share=6.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:50:00] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6824 0.001514 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:50:05] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6824 0.001315 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:50:11] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6824 0.002327 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'e96d406cc38eaf473eb70a5cd186388b130100649492589ca5090633584325a6', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1174789} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '02dca31319a3b8f5c3ce3ce9dc593e6f4c0bf04cc7c192321c26a8fd21f971a7', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1174810} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55592328, partition_space_available_1k=25498156, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1347928, cpu_share=0.16, mem_res=1334020, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=217760, cpu_share=0.02, mem_res=172728, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44848, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18721324, free=3343024, buffers=848272, cached=10217488, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=6.69, five_min_avg=5.41, fifteen_min_avg=4.05, cpu_share=4.88, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'bfa93f8ac75a448823c2a89b991aa2b7ac99ed3df697ac829ed2785e00dd5c48', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1176869} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '5ebaeaadb795f3fb4e40e3aa25b51a99d5f92e1bb4561e17657d48f55b300ad4', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1176890} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55603676, partition_space_available_1k=25486808, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1351552, cpu_share=0.1, mem_res=1337224, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=220004, cpu_share=0.0, mem_res=174948, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44860, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18385308, free=3652536, buffers=848316, cached=10243380, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.9, five_min_avg=5.47, fifteen_min_avg=4.15, cpu_share=4.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '339d67d951f00d4c66f63f45a230544b9b26fef6ac8e9a89df72ea5fef0c4e9e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1178307} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '594673a20f79000a6521f9ce7d4dbca5837d8359f1300d60717649a08251627f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1178328} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55625296, partition_space_available_1k=25465188, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1351680, cpu_share=0.14, mem_res=1337200, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=252680, cpu_share=0.01, mem_res=207520, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44864, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18455240, free=3550164, buffers=848332, cached=10275412, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=6.39, five_min_avg=5.9, fifteen_min_avg=4.4, cpu_share=4.88, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'da34e92ac191b130bb606b88ca86e82ea89e4e72f0806677c7d7faf17207cdf1', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1179579} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '00c2fb7ebbd21cdff3b82256056292a246b4daa12a8045499df5cbc853ed8fcf', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1179601} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55640244, partition_space_available_1k=25450240, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355692, cpu_share=0.11, mem_res=1340512, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=238212, cpu_share=0.0, mem_res=193040, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44860, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18968160, free=3018024, buffers=848360, cached=10293532, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.38, five_min_avg=5.68, fifteen_min_avg=4.42, cpu_share=4.88, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '9b8f180ab98cae6bf260e5410585c85487dd7f8fd4fb02695a4509ffb81ea25e', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1184774} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '2eb3345f98d3daebb41ff92311b0d0bb65b19fafd1b533a4d895df908a802e5f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1185066} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55717392, partition_space_available_1k=25373092, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1355852, cpu_share=0.09, mem_res=1340536, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=240276, cpu_share=0.0, mem_res=195088, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44868, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=19660980, free=2276908, buffers=848388, cached=10342296, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=11.01, five_min_avg=7.08, fifteen_min_avg=4.98, cpu_share=6.38, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:55:17] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6825 0.001570 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:55:22] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6825 0.001444 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:55:28] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6825 0.001343 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '5b6ab32d9f66751c4d23b4c3b29a91b3d3429eb666c960e5bcd5dd62b841105f', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1204854} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '12030bd1ac16b2f4f11404501597685f13d95e6c70a78234599b7e4d659b5fbc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1204881} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55705456, partition_space_available_1k=25385028, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356000, cpu_share=0.12, mem_res=1340468, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=242796, cpu_share=0.01, mem_res=197584, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44872, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18770812, free=3146496, buffers=848808, cached=10362348, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=6.5, five_min_avg=6.5, fifteen_min_avg=4.9, cpu_share=5.38, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '49bab63420d8305ee287db42e0e2fef5448053fce485adf025632486253cbb22', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1206078} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'e1c3e367b72ad7d854024f386f194dceef4b7352813ba43b691b612f1acd9e58', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1206102} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55721172, partition_space_available_1k=25369312, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1356144, cpu_share=0.07, mem_res=1340540, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=244000, cpu_share=0.0, mem_res=198776, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44876, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18918200, free=2979440, buffers=848812, cached=10381968, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.13, five_min_avg=6.09, fifteen_min_avg=4.86, cpu_share=4.62, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 01:57:13] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6824 0.001683 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '8e705741a9c49531137cc9eb04f08cf559b5278152fff0a7e69b8cf43f8b30cc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1208496} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': '922a611422660a2d6a67d6e4db85cc9361a40e018425b4b31af127fbc4396893', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1208517} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55731460, partition_space_available_1k=25359024, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1373460, cpu_share=0.09, mem_res=1357796, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=245776, cpu_share=0.0, mem_res=200540, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44884, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18893964, free=2990020, buffers=848864, cached=10394464, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.55, five_min_avg=5.78, fifteen_min_avg=4.83, cpu_share=4.38, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '948825e995f54950ec88152bab91964fc9d0622c1894e302060ee4bbcb55e094', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1210013} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'cbb1b3835a68c657204d881cb54994129ed3a9ec983722e93c15e4061b495474', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1210034} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55741368, partition_space_available_1k=25349116, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1374844, cpu_share=0.06, mem_res=1358580, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=247356, cpu_share=0.01, mem_res=202104, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44892, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18594460, free=3275064, buffers=848868, cached=10408028, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=5.54, five_min_avg=5.86, fifteen_min_avg=4.92, cpu_share=4.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': 'c8ffd55da50cbe2f1a75a694c7aee0fc33ac2e407afc17626c44d13b599671fc', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1211063} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'c2d9bb944420bce88e009ef27457a36ceb9f8080e15a3c375726366ae6b5971d', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1211084} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55751128, partition_space_available_1k=25339356, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1374932, cpu_share=0.07, mem_res=1358588, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=248508, cpu_share=0.0, mem_res=203248, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44892, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18609660, free=3248072, buffers=848876, cached=10420120, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=4.01, five_min_avg=5.37, fifteen_min_avg=4.8, cpu_share=3.75, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo') DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 02:00:08] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6823 0.001770 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 02:00:13] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6823 0.001444 DEBUG:contrail-config-database-nodemgr:send sync_uve in the [ObjectConfigDatabaseInfo:NodeStatusUVE] map [Introspect:8112]10.0.0.129 - - [2025-10-13 02:00:19] "GET /Snh_SandeshUVECacheReq?x=NodeStatus HTTP/1.1" 200 6823 0.001910 DEBUG:contrail-config-database-nodemgr:dir for config is /var/lib/cassandra/data CRITICAL:root:Exit data: {'ID': '719b0794ee10cf241699ba97e3e6b2b8aa9b10fb2e5ec98ecfd624df0827145a', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'cqlsh', 'arguments': ['--ssl', '10.0.0.129', '9041', '-e', 'quit'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1234520} WARNING:contrail-config-database-nodemgr:CQLSH failed, but Cassandra native transport port is open CRITICAL:root:Exit data: {'ID': 'ff61db0927929f79ea93f6cc2951e3b0bb3d59590acd3a6bdec86862bb88c450', 'Running': False, 'ExitCode': 1, 'ProcessConfig': {'tty': True, 'entrypoint': 'nodetool', 'arguments': ['-p', '7201', 'compactionstats'], 'privileged': False}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'fa1853320e4217fe0a24a5a42eabe85f267f91722781538ab1a6d43f31c90734', 'DetachKeys': '', 'Pid': 1234543} ERROR:contrail-config-database-nodemgr:Failed to get nodetool compactionstats: Result: 1 DEBUG:contrail-config-database-nodemgr:DBG: event_tick_60: node_status=NodeStatus(name='cn-jenkins-deploy-platform-juju-os-589-1', deleted=None, status=None, process_status=None, process_info=None, disk_usage_info={'/dev/vda1': DiskPartitionUsageStats(partition_type='ext4', partition_name=None, partition_space_used_1k=55780884, partition_space_available_1k=25309600, percentage_partition_space_used=69, _LinuxSysData__key='/dev/vda1')}, description=None, all_core_file_list=None, build_info=None, process_mem_cpu_usage={'cassandra': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=1375136, cpu_share=0.09, mem_res=1358592, _EventManager__key='cassandra'), 'zookeeper': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=250636, cpu_share=0.0, mem_res=205356, _EventManager__key='zookeeper'), 'contrail-config-database-nodemgr': ProcessCpuInfo(module_id=None, inst_id=None, mem_virt=44900, cpu_share=0.01, mem_res=42704, _EventManager__key='contrail-config-database-nodemgr')}, system_cpu_info=None, system_mem_usage=SysMemInfo(total=32665120, used=18415148, free=3407096, buffers=848884, cached=10455680, node_type='config-database-node'), system_cpu_usage=SysCpuInfo(one_min_avg=9.74, five_min_avg=6.77, fifteen_min_avg=5.32, cpu_share=6.5, node_type='config-database-node'), running_package_version=None, installed_package_version=None, _table='ObjectConfigDatabaseInfo')