cache filled mbuf to reduce 2K mbuf usage

DAYceng · Mar 16, 2023 · 05148f4 · 05148f4
1 parent 7d043f4
commit 05148f4
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 7 deletions.
diff --git a/scripts/automation/trex_control_plane/interactive/trex/common/trex_client.py b/scripts/automation/trex_control_plane/interactive/trex/common/trex_client.py
@@ -1993,7 +1993,9 @@ def set_global_cfg (self, params):
             :parameters:
                 params: dictionary
                     dictionary of global configuration parameters. Available parameters:
-                    "sched_max_stretch", type = double, default =0.0, units = usec, scheduler's maximum time for stretch in case it is a high value there won't be a scheduler compensation on burst and time will not be stretch. 0.0 means use the default internal value (~100usec). value could not be lower than 100usec will be considered as 100usec.
+                    "sched_max_stretch", type = double, default = 0.0, units = usec, scheduler's maximum time for stretch in case it is a high value there won't be a scheduler compensation on burst and time will not be stretch. 0.0 means use the default internal value (~100usec). value could not be lower than 100usec will be considered as 100usec. The change cannot be applied to the current epoch.
+                    "process_at_cp", type = bool, default = False, ASTF profile needs to prepare for traffic generation. By default, it is performed at DP cores. But when you run multiple profiles, since it consumes DP processing power, it may affect the traffic generation. This parameter can remove the process from DP cores and perform it at CP core.
+                    "do_mbuf_cache", type = bool, default = False, ASTF profile's buffer can be filled with specific pattern. This parameter enables the cached mbuf for the same patterns. It will save 2K mbuf consumption when you run lots of profiles with the same patterns. But, since a pattern consumes a 2K mbuf permanently, please enable this parameter only the patterns are limited.
 
             :raises:
                 + :exc:`TRexError`
@@ -2012,6 +2014,14 @@ def set_global_cfg (self, params):
 
     @client_api('command', True)
     def get_global_cfg (self):
+        """
+            Get global configuration parameter values. See `set_global_cfg` for the parameters.
+
+            :raises:
+                + :exc:`TRexError`
+
+        """
+
         rc = self._transmit("get_global_cfg")
         if not rc:
             raise TRexError(rc)

diff --git a/src/44bsd/tcp_socket.cpp b/src/44bsd/tcp_socket.cpp
@@ -1100,19 +1100,54 @@ int utl_mbuf_buffer_create_and_copy(uint8_t socket,
 }
 
 
+/*
+ * The mbuf filled with patterns can be cached for other buffers.
+ * It will reduce 2K mbuf usage when lots of buffers are loaded.
+ * To avoid the 16bit refcnt overflow, const-mbuf is used.
+ * The const-mbuf could not be freed because it will stay in the driver's TX descriptor.
+ * In general, the number of cached mbufs is small and has no issue.
+ */
+#define MAX_MBUF_CACHE  32
+static std::map<std::string,rte_mbuf_t*> mbufs_cached[MAX_SOCKETS_SUPPORTED];
+static std::mutex mbufs_cached_mutex[MAX_SOCKETS_SUPPORTED];
+
+static rte_mbuf_t* get_mbuf_cached(uint8_t socket, std::string& fstring) {
+    if (CGlobalInfo::m_do_mbuf_cache) {
+        std::unique_lock<std::mutex> my_lock(mbufs_cached_mutex[socket]);
+
+        auto it = mbufs_cached[socket].find(fstring);
+        if (it != mbufs_cached[socket].end()) {
+            return it->second;
+        }
+    }
+    return nullptr;
+}
+
+static void set_mbuf_cached(uint8_t socket, std::string& fstring, rte_mbuf_t* mbuf) {
+    if (CGlobalInfo::m_do_mbuf_cache && (mbufs_cached[socket].size() < MAX_MBUF_CACHE)) {
+        std::unique_lock<std::mutex> my_lock(mbufs_cached_mutex[socket]);
+
+        if (mbufs_cached[socket].find(fstring) == mbufs_cached[socket].end()) {
+            mbufs_cached[socket][fstring] = mbuf;
+            rte_mbuf_set_as_core_const(mbuf);
+        }
+    }
+}
+
+
 int utl_mbuf_buffer_create_and_copy(uint8_t socket,
                                     CMbufBuffer * buf,
                                     uint32_t blk_size,
                                     uint8_t *d,
                                     uint32_t d_size,
                                     uint32_t size,
-                                    uint8_t *f,
-                                    uint32_t f_size,
+                                    std::string& fill_string,
                                     bool mbuf_const){
 
     buf->Create(blk_size);
     uint8_t cnt = 0;
-    rte_mbuf_t* mbuf_fill = nullptr;
+    rte_mbuf_t* mbuf_fill = get_mbuf_cached(socket, fill_string);
+
     while (size>0) {
         uint32_t alloc_size = bsd_umin(blk_size,size);
         auto copy_size = bsd_umin(d_size, alloc_size);
@@ -1131,7 +1166,14 @@ int utl_mbuf_buffer_create_and_copy(uint8_t socket,
                 d_size -= copy_size;
             } else if (!mbuf_fill) {
                 mbuf_fill = m; // filled data reference mbuf
+                if (alloc_size == blk_size) {
+                    set_mbuf_cached(socket, fill_string, mbuf_fill);
+                }
             }
+
+            uint8_t* f = (uint8_t*)fill_string.c_str();
+            uint32_t f_size = fill_string.size();
+
             for (auto i = 0; i < (alloc_size-copy_size); i++ ) {
                 *p = (f_size>0) ? f[cnt%f_size]: cnt;
                 cnt++;

diff --git a/src/44bsd/tcp_socket.h b/src/44bsd/tcp_socket.h
@@ -1023,8 +1023,7 @@ int utl_mbuf_buffer_create_and_copy(uint8_t socket,
                                     uint8_t *d,
                                     uint32_t d_size,
                                     uint32_t size,
-                                    uint8_t *f,
-                                    uint32_t f_size,
+                                    std::string& fill_string,
                                     bool mbuf_const=false);
 
 

diff --git a/src/astf/astf_db.cpp b/src/astf/astf_db.cpp
@@ -1629,7 +1629,7 @@ bool CAstfDB::convert_bufs(uint8_t socket_id) {
             if (buf_obj["fill"] != Json::nullValue) {
                 fill_data = base64_decode(buf_obj["fill"].asString());
             }
-            utl_mbuf_buffer_create_and_copy(socket_id, tcp_buf, 2048, (uint8_t *)(temp_str.c_str()), temp_str.size(),buf_len,(uint8_t*)(fill_data.c_str()),fill_data.size(),true);
+            utl_mbuf_buffer_create_and_copy(socket_id, tcp_buf, 2048, (uint8_t *)(temp_str.c_str()), temp_str.size(),buf_len,fill_data,true);
         } else {
             json_buf = m_val["buf_list"][buf_index].asString();
             temp_str = base64_decode(json_buf);

diff --git a/src/stx/common/trex_rpc_cmds_common.cpp b/src/stx/common/trex_rpc_cmds_common.cpp
@@ -1740,6 +1740,9 @@ TrexRpcCmdSetGlobalCfg::_run(const Json::Value &params, Json::Value &result) {
     if (params["process_at_cp"] != Json::Value::null) {
         CGlobalInfo::m_process_at_cp = parse_bool(params, "process_at_cp", result);
     }
+    if (params["do_mbuf_cache"] != Json::Value::null) {
+        CGlobalInfo::m_do_mbuf_cache = parse_bool(params, "do_mbuf_cache", result);
+    }
     return (TREX_RPC_CMD_OK);
 }
 
@@ -1749,6 +1752,7 @@ TrexRpcCmdGetGlobalCfg::_run(const Json::Value &params, Json::Value &result) {
 
     section["sched_max_stretch"] = CGlobalInfo::m_burst_offset_dtime;
     section["process_at_cp"] = CGlobalInfo::m_process_at_cp;
+    section["do_mbuf_cache"] = CGlobalInfo::m_do_mbuf_cache;
 
     return (TREX_RPC_CMD_OK);
 }

diff --git a/src/trex_global.cpp b/src/trex_global.cpp
@@ -28,6 +28,7 @@ CRteMemPool         CGlobalInfo::m_mem_pool[MAX_SOCKETS_SUPPORTED];
 uint32_t            CGlobalInfo::m_nodes_pool_size = 10*1024;
 double              CGlobalInfo::m_burst_offset_dtime;
 bool                CGlobalInfo::m_process_at_cp = false;
+bool                CGlobalInfo::m_do_mbuf_cache = false;
 CParserOption       CGlobalInfo::m_options;
 CGlobalMemory       CGlobalInfo::m_memory_cfg;
 CPlatformSocketInfo CGlobalInfo::m_socket;

diff --git a/src/trex_global.h b/src/trex_global.h
@@ -1100,6 +1100,7 @@ class CGlobalInfo {
     static uint32_t              m_nodes_pool_size;
     static double                m_burst_offset_dtime;
     static bool                  m_process_at_cp;
+    static bool                  m_do_mbuf_cache;
     static CParserOption         m_options;
     static CGlobalMemory         m_memory_cfg;
     static CPlatformSocketInfo   m_socket;