From 00445404fff6d3673464e2a5defca1dc97ab3a43 Mon Sep 17 00:00:00 2001 From: bhaveshdell <55215495+bhaveshdell@users.noreply.github.com> Date: Sat, 11 May 2024 00:14:54 -0700 Subject: [PATCH 1/6] Add new tables for Event and Alarms. (#852) Why I did it This PR contains code changes for providing extension to the Event Framework as specified in the https://github.com/sonic-net/SONiC/pull/1409 How I did it Followed design specified in https://github.com/sonic-net/SONiC/pull/1409. Add new tables for events and alarms. --- common/schema.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/schema.h b/common/schema.h index d254d5049..e46a711de 100644 --- a/common/schema.h +++ b/common/schema.h @@ -23,6 +23,7 @@ namespace swss { #define CHASSIS_APP_DB 12 #define CHASSIS_STATE_DB 13 #define APPL_STATE_DB 14 +#define EVENT_DB 19 /***** APPLICATION DATABASE *****/ @@ -567,6 +568,12 @@ namespace swss { #define APP_FABRIC_MONITOR_DATA_TABLE_NAME "FABRIC_MONITOR_TABLE" #define APP_FABRIC_MONITOR_PORT_TABLE_NAME "FABRIC_PORT_TABLE" +#define EVENT_HISTORY_TABLE_NAME "EVENT" +#define EVENT_CURRENT_ALARM_TABLE_NAME "ALARM" +#define EVENT_STATS_TABLE_NAME "EVENT_STATS" +#define EVENT_ALARM_STATS_TABLE_NAME "ALARM_STATS" + + #ifdef __cplusplus } #endif From 3750752dfaf68d3d2cecf49ad77cdadde3c9a955 Mon Sep 17 00:00:00 2001 From: Liu Shilong Date: Mon, 27 May 2024 20:41:41 +0800 Subject: [PATCH 2/6] [ci] Use requests==2.31.0 instead of latest version to avoid test failure. (#877) Python package requests' latest release 2.32.0 has issues with docker package. Use old stable version 2.31.0 instead. --- .azure-pipelines/test-docker-sonic-vs-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 7c0998fba..f26ee4c09 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -64,7 +64,7 @@ jobs: # install packages for vs test sudo apt-get install -y net-tools bridge-utils vlan sudo apt-get install -y python3-pip - sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0 + sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0 requests==2.31.0 displayName: "Install dependencies" - script: | From 138b32ec7e56f2021a079c1bcbee11f578cf6ac0 Mon Sep 17 00:00:00 2001 From: Ze Gan Date: Wed, 29 May 2024 00:34:12 +0800 Subject: [PATCH 3/6] [swig]: Fix swig template memory leak on issue 17025 (#876) Fix the issue sonic-net/sonic-buildimage#17025 about Redis set activity Description The issue reports a memory leak on the Redis set operations Reason Didn't decrease the reference count after PySequence_GetItem Use the inappropriate Swig API and didn't release the string of SWIG_AsPtr_std_string Fix: Refer PR: Fix swig template memory leak #859 from @praveenraja1 Replace the API SWIG_AsPtr_std_string to SWIG_AsVal_std_string Add unit test To monitor there is no dramatic memory increment after a huge amount of Redis set operations. Signed-off-by: Ze Gan --- .azure-pipelines/build-template.yml | 2 +- pyext/swsscommon.i | 39 ++++++++++++++++++----------- tests/test_redis_ut.py | 28 +++++++++++++++++++++ 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index f2a0ee496..d1813c23a 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -110,7 +110,7 @@ jobs: displayName: "Install gcovr 5.2 (for --exclude-throw-branches support)" - script: | set -ex - sudo pip install Pympler==0.8 + sudo pip install Pympler==0.8 pytest sudo apt-get install -y redis-server sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 81b9cdb19..2bf953b11 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -56,6 +56,8 @@ #include "zmqclient.h" #include "zmqconsumerstatetable.h" #include "zmqproducerstatetable.h" +#include +#include %} %include @@ -156,31 +158,36 @@ SWIG_Python_AppendOutput($result, temp); } -%typemap(in, fragment="SWIG_AsPtr_std_string") +%typemap(in, fragment="SWIG_AsVal_std_string") const std::vector,std::allocator< std::pair< std::string,std::string > > > & (std::vector< std::pair< std::string,std::string >,std::allocator< std::pair< std::string,std::string > > > temp, int res) { res = SWIG_OK; for (int i = 0; i < PySequence_Length($input); ++i) { temp.push_back(std::pair< std::string,std::string >()); - PyObject *item = PySequence_GetItem($input, i); - if (!PyTuple_Check(item) || PyTuple_Size(item) != 2) { + std::unique_ptr > item( + PySequence_GetItem($input, i), + [](PyObject *ptr){ + Py_DECREF(ptr); + }); + if (!PyTuple_Check(item.get()) || PyTuple_Size(item.get()) != 2) { SWIG_fail; } - PyObject *key = PyTuple_GetItem(item, 0); - PyObject *value = PyTuple_GetItem(item, 1); - std::string *ptr = (std::string *)0; + PyObject *key = PyTuple_GetItem(item.get(), 0); + PyObject *value = PyTuple_GetItem(item.get(), 1); + std::string str; + if (PyBytes_Check(key)) { temp.back().first.assign(PyBytes_AsString(key), PyBytes_Size(key)); - } else if (SWIG_AsPtr_std_string(key, &ptr)) { - temp.back().first = *ptr; + } else if (SWIG_AsVal_std_string(key, &str) != SWIG_ERROR) { + temp.back().first = str; } else { SWIG_fail; } if (PyBytes_Check(value)) { temp.back().second.assign(PyBytes_AsString(value), PyBytes_Size(value)); - } else if (SWIG_AsPtr_std_string(value, &ptr)) { - temp.back().second = *ptr; + } else if (SWIG_AsVal_std_string(value, &str) != SWIG_ERROR) { + temp.back().second = str; } else { SWIG_fail; } @@ -191,13 +198,17 @@ %typemap(typecheck) const std::vector< std::pair< std::string,std::string >,std::allocator< std::pair< std::string,std::string > > > &{ $1 = 1; for (int i = 0; i < PySequence_Length($input); ++i) { - PyObject *item = PySequence_GetItem($input, i); - if (!PyTuple_Check(item) || PyTuple_Size(item) != 2) { + std::unique_ptr > item( + PySequence_GetItem($input, i), + [](PyObject *ptr){ + Py_DECREF(ptr); + }); + if (!PyTuple_Check(item.get()) || PyTuple_Size(item.get()) != 2) { $1 = 0; break; } - PyObject *key = PyTuple_GetItem(item, 0); - PyObject *value = PyTuple_GetItem(item, 1); + PyObject *key = PyTuple_GetItem(item.get(), 0); + PyObject *value = PyTuple_GetItem(item.get(), 1); if (!PyBytes_Check(key) && !PyUnicode_Check(key) && !PyString_Check(key) diff --git a/tests/test_redis_ut.py b/tests/test_redis_ut.py index b88a3539c..bf395a34f 100644 --- a/tests/test_redis_ut.py +++ b/tests/test_redis_ut.py @@ -1,5 +1,6 @@ import os import time +import psutil import pytest import multiprocessing from threading import Thread @@ -851,3 +852,30 @@ def test_SmartSwitchDBConnector(): assert tbl.get("dputest2")[1][1] == ("dashfield2", "dashvalue2") assert len(SonicDBConfig.getDbKeys()) == len(global_db_config_json["INCLUDES"]) + +def test_TableSetBinary(): + app_db = swsscommon.DBConnector("APPL_DB", 0, True) + t = swsscommon.Table(app_db, "TABLE") + buff = b"" + for i in range(0, 256): + buff += bytes([i]) + buff = buff.decode('latin-1') + fvs = swsscommon.FieldValuePairs([("binary", buff)]) + t.set("binary", fvs) + (status, fvs) = t.get("binary") + assert status == True + assert fvs[0][1] == buff + + +def test_TableOpsMemoryLeak(): + OP_COUNT = 50000 + app_db = swsscommon.DBConnector("APPL_DB", 0, True) + t = swsscommon.Table(app_db, "TABLE") + long_data = "x" * 100 + fvs = swsscommon.FieldValuePairs([(long_data, long_data)]) + rss = psutil.Process(os.getpid()).memory_info().rss + for _ in range(OP_COUNT): + t.set("long_data", fvs) + t.get("long_data") + assert psutil.Process(os.getpid()).memory_info().rss - rss < OP_COUNT + From 391e27bd6769fa64d483c2c67a1079c174f4975f Mon Sep 17 00:00:00 2001 From: Feng-msft Date: Fri, 31 May 2024 08:05:58 +0800 Subject: [PATCH 4/6] Add new table schema for BMP feature. (#879) --- common/schema.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/schema.h b/common/schema.h index e46a711de..46f75d07a 100644 --- a/common/schema.h +++ b/common/schema.h @@ -24,6 +24,7 @@ namespace swss { #define CHASSIS_STATE_DB 13 #define APPL_STATE_DB 14 #define EVENT_DB 19 +#define BMP_STATE_DB 20 /***** APPLICATION DATABASE *****/ @@ -323,6 +324,7 @@ namespace swss { #define CFG_BGP_MONITORS_TABLE_NAME "BGP_MONITORS" #define CFG_BGP_PEER_RANGE_TABLE_NAME "BGP_PEER_RANGE" #define CFG_BGP_DEVICE_GLOBAL_TABLE_NAME "BGP_DEVICE_GLOBAL" +#define CFG_BMP_TABLE_NAME "BMP" #define CFG_SWITCH_HASH_TABLE_NAME "SWITCH_HASH" #define CFG_DEVICE_METADATA_TABLE_NAME "DEVICE_METADATA" @@ -573,6 +575,10 @@ namespace swss { #define EVENT_STATS_TABLE_NAME "EVENT_STATS" #define EVENT_ALARM_STATS_TABLE_NAME "ALARM_STATS" +/***** BMP STATE DATABASE *****/ +#define BMP_STATE_BGP_NEIGHBOR_TABLE "BGP_NEIGHBOR_TABLE" +#define BMP_STATE_BGP_RIB_IN_TABLE "BGP_RIB_IN_TABLE" +#define BMP_STATE_BGP_RIB_OUT_TABLE "BGP_RIB_OUT_TABLE" #ifdef __cplusplus } From e37bfea64fb9e1f40f7c9b6ad759e92f5f15bec9 Mon Sep 17 00:00:00 2001 From: arista-nwolfe <94405414+arista-nwolfe@users.noreply.github.com> Date: Tue, 11 Jun 2024 20:10:38 -0400 Subject: [PATCH 5/6] Don't use unix socket on for redis_chassis.server (#873) --- sonic-db-cli/sonic-db-cli.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic-db-cli/sonic-db-cli.cpp b/sonic-db-cli/sonic-db-cli.cpp index 027c810cc..8f9711506 100755 --- a/sonic-db-cli/sonic-db-cli.cpp +++ b/sonic-db-cli/sonic-db-cli.cpp @@ -139,14 +139,14 @@ int executeCommands( try { int db_id = SonicDBConfig::getDbId(db_name, netns); - if (useUnixSocket) + auto host = SonicDBConfig::getDbHostname(db_name, netns); + if (useUnixSocket && host != "redis_chassis.server") { auto db_socket = SonicDBConfig::getDbSock(db_name, netns); client = make_shared(db_id, db_socket, 0); } else { - auto host = SonicDBConfig::getDbHostname(db_name, netns); auto port = SonicDBConfig::getDbPort(db_name, netns); client = make_shared(db_id, host, port, 0); } From aba0f66dfe1c77c6256cd6be6d2edeefa611efa4 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 13 Aug 2024 06:56:04 -0700 Subject: [PATCH 6/6] Add new DASH APPL_DB table names (#897) --- common/schema.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/schema.h b/common/schema.h index 46f75d07a..4007967ae 100644 --- a/common/schema.h +++ b/common/schema.h @@ -166,6 +166,11 @@ namespace swss { #define APP_DASH_ROUTE_TABLE_NAME "DASH_ROUTE_TABLE" #define APP_DASH_ROUTE_RULE_TABLE_NAME "DASH_ROUTE_RULE_TABLE" #define APP_DASH_VNET_MAPPING_TABLE_NAME "DASH_VNET_MAPPING_TABLE" +#define APP_DASH_ENI_ROUTE_TABLE_NAME "DASH_ENI_ROUTE_TABLE" +#define APP_DASH_ROUTE_GROUP_TABLE_NAME "DASH_ROUTE_GROUP_TABLE" +#define APP_DASH_TUNNEL_TABLE_NAME "DASH_TUNNEL_TABLE" +#define APP_DASH_PA_VALIDATION_TABLE_NAME "DASH_PA_VALIDATION_TABLE" +#define APP_DASH_ROUTING_APPLIANCE_TABLE_NAME "DASH_ROUTING_APPLIANCE_TABLE" /***** TO BE REMOVED *****/