Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
       3             :  * Copyright Contributors to the OpenSDN project
       4             :  */
       5             : 
       6             : #include <sys/socket.h>
       7             : #if defined(__linux__)
       8             : #include <linux/netlink.h>
       9             : #endif
      10             : #include <fcntl.h>
      11             : #include <sys/mman.h>
      12             : #include <sys/types.h>
      13             : #include <sys/ipc.h>
      14             : #include <sys/shm.h>
      15             : #include <asm/types.h>
      16             : #include <boost/asio.hpp>
      17             : 
      18             : #include <base/timer.h>
      19             : #include <base/task_trigger.h>
      20             : #include <base/address_util.h>
      21             : #include <cmn/agent_cmn.h>
      22             : #include <services/services_init.h>
      23             : #include <uve/stats_collector.h>
      24             : #include <services/icmp_error_proto.h>
      25             : #include <pkt/flow_proto.h>
      26             : #include <ksync/ksync_index.h>
      27             : #include <ksync/ksync_entry.h>
      28             : #include <ksync/ksync_object.h>
      29             : #include <ksync/ksync_netlink.h>
      30             : #include <ksync/ksync_sock.h>
      31             : #include <ksync/ksync_sock_user.h>
      32             : 
      33             : #include <vr_types.h>
      34             : #include <nl_util.h>
      35             : #include <vr_flow.h>
      36             : #include <ini_parser.h>
      37             : #include <vr_genetlink.h>
      38             : 
      39             : #include "ksync_init.h"
      40             : #include "ksync_flow_memory.h"
      41             : #include "sandesh_ksync.h"
      42             : #include "init/agent_param.h"
      43             : 
      44             : using namespace boost::asio::ip;
      45             : static const int kTestFlowTableSize = 131072 * sizeof(vr_flow_entry);
      46             : 
      47           4 : KSyncMemory::KSyncMemory(KSync *ksync, uint32_t minor_id) :
      48           4 :     ksync_(ksync),
      49           4 :     table_path_(),
      50           4 :     major_devid_(0),
      51           4 :     minor_devid_(minor_id),
      52           4 :     table_size_(0),
      53           4 :     table_entries_count_(0),
      54           4 :     audit_timer_(TimerManager::CreateTimer
      55           4 :                  (*(ksync->agent()->event_manager())->io_service(),
      56             :                   " Audit Timer",
      57             :                   ksync->agent()->task_scheduler()->GetTaskId(kTaskFlowAudit),
      58             :                   0)),
      59           4 :     audit_timeout_(0),
      60           4 :     audit_yield_(0),
      61           4 :     audit_interval_(0),
      62           4 :     audit_idx_(0),
      63           8 :     audit_list_() {
      64           4 : }
      65             : 
      66           4 : KSyncMemory::~KSyncMemory() {
      67           4 :     TimerManager::DeleteTimer(audit_timer_);
      68           4 : }
      69             : 
      70           0 : void KSyncMemory::Init() {
      71           0 :     audit_interval_ = kAuditYieldTimer;
      72           0 :     audit_timeout_ = kAuditTimeout;
      73           0 :     uint32_t table_count = table_entries_count_;
      74             :     // Compute number of entries to visit per timer interval so that complete
      75             :     // table can be visited in kAuditSweepTime
      76           0 :     uint32_t timer_per_sec = 1000 / kAuditYieldTimer;
      77           0 :     uint32_t timer_per_sweep = kAuditSweepTime * timer_per_sec;
      78           0 :     audit_yield_ = table_count / timer_per_sweep;
      79           0 :     if (audit_yield_ > kAuditYieldMax)
      80           0 :         audit_yield_ = kAuditYieldMax;
      81           0 :     if (audit_yield_ < kAuditYieldMin)
      82           0 :         audit_yield_ = kAuditYieldMin;
      83             : 
      84           0 :     audit_timer_->Start(audit_interval_,
      85             :                         boost::bind(&KSyncMemory::AuditProcess, this));
      86           0 : }
      87             : 
      88           0 : void KSyncMemory::Mmap(bool unlink_node, void *khpmem, bool kernel_mode) {
      89             :     // In case of non hugepage kernel mode,
      90             :     // Remove the existing /dev/ file first. We will add it again in vr_table_map
      91           0 :     if (!khpmem && unlink_node) {
      92           0 :         const char *error_msg = vr_table_unlink(table_path_.c_str());
      93           0 :         if (error_msg) {
      94           0 :             LOG(ERROR, "Error unmapping KSync memory: " << error_msg);
      95           0 :             assert(0);
      96             :         }
      97             :     }
      98           0 :     parse_ini_file();
      99             : 
     100             :     // Kernel hugepage present
     101           0 :     if (khpmem) {
     102           0 :         table_ = khpmem;
     103             :     } else {
     104             :         // DPDK or kernel without hugepage support
     105             :         const char *table_str;
     106           0 :         if (kernel_mode) {
     107           0 :             table_str = NULL;
     108             :         } else {
     109           0 :             table_str = table_path_.c_str();
     110             :         }
     111           0 :         const char *mmap_error_msg = vr_table_map(major_devid_, minor_devid_, table_str,
     112           0 :                                                   table_size_, &table_);
     113           0 :         if (mmap_error_msg) {
     114           0 :             LOG(ERROR, "Error mapping KSync memory. Device: " << table_path_ << "; " << mmap_error_msg);
     115           0 :             assert(0);
     116             :         }
     117           0 :         LOG(INFO, "Mem mapped dev file:" << table_path_.c_str() << " to addr:" << table_ << "\n");
     118             :     }
     119             : 
     120           0 :     table_entries_count_ = table_size_ / get_entry_size();
     121           0 :     SetTableSize();
     122           0 : }
     123             : 
     124           0 : int KSyncMemory::GetKernelTableSize() {
     125             :     struct nl_client *cl;
     126             :     int attr_len;
     127             :     int encode_len, ret;
     128             : 
     129           0 :     assert((cl = nl_register_client()) != NULL);
     130             : 
     131           0 :     assert(nl_socket(cl, AF_NETLINK, SOCK_DGRAM, NETLINK_GENERIC) > 0);
     132           0 :     assert(nl_connect(cl, 0, 0, 0) == 0);
     133             : 
     134           0 :     assert(vrouter_obtain_family_id(cl) > 0);
     135             : 
     136           0 :     assert(nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST) == 0);
     137           0 :     assert(nl_build_genlh(cl, SANDESH_REQUEST, 0) == 0);
     138             : 
     139           0 :     attr_len = nl_get_attr_hdr_size();
     140             : 
     141           0 :     encode_len = EncodeReq(cl, attr_len);
     142           0 :     nl_build_attr(cl, encode_len, NL_ATTR_VR_MESSAGE_PROTOCOL);
     143           0 :     nl_update_nlh(cl);
     144             : 
     145           0 :     if ((ret = nl_sendmsg(cl)) < 0) {
     146           0 :         LOG(DEBUG, "Error requesting  Table message. Error : " << ret);
     147           0 :         assert(0);
     148             :     }
     149             : 
     150           0 :     while ((ret = nl_recvmsg(cl)) > 0) {
     151           0 :         KSyncSockNetlink::NetlinkDecoder(cl->cl_buf,
     152           0 :                                          KSyncSock::GetAgentSandeshContext(0));
     153             :     }
     154           0 :     nl_free_client(cl);
     155           0 :     return table_size_;
     156             : }
     157             : 
     158             : // Steps to map  table entry
     159             : // In case of non huge pages
     160             : // - Query the  table parameters from kernel
     161             : // - Create device /dev/ with major-num and minor-num
     162             : // - Map device memory
     163             : // In case of huge pages
     164             : // - Just use the huge page memory which is initialized
     165             : //   and passed to this function
     166           0 : void KSyncMemory::InitMem(void *hpmem) {
     167           0 :     GetKernelTableSize();
     168           0 :     Mmap(true, hpmem, true);
     169           0 :     return;
     170             : }
     171             : 
     172           0 : void KSyncMemory::InitTest() {
     173           0 :     assert(0);
     174             : }
     175             : 
     176           0 : void KSyncMemory::Shutdown() {
     177           0 :     assert(0);
     178             : }
     179             : 
     180           0 : bool KSyncMemory::AuditProcess() {
     181             :     // Get current time
     182           0 :     uint64_t t = UTCTimestampUsec();
     183             : 
     184           0 :     while (!audit_list_.empty()) {
     185           0 :         AuditEntry list_entry = audit_list_.front();
     186             :         // audit_list_ is sorted on last time of insertion in the list
     187             :         // So, break on finding first  entry that cannot be aged
     188           0 :         if ((t - list_entry.timeout) < audit_timeout_) {
     189             :             /* Wait for audit_timeout_ to create short  for the entry */
     190           0 :             break;
     191             :         }
     192           0 :         uint32_t idx = list_entry.audit_idx;
     193           0 :         uint32_t gen_id = list_entry.audit_gen_id;
     194           0 :         audit_list_.pop_front();
     195           0 :         DecrementHoldFlowCounter();
     196           0 :         CreateProtoAuditEntry(idx, gen_id);
     197             :     }
     198             : 
     199           0 :     uint32_t count = 0;
     200             :     uint8_t gen_id;
     201           0 :     assert(audit_yield_);
     202           0 :     while (count < audit_yield_) {
     203           0 :         if (IsInactiveEntry(audit_idx_, gen_id)) {
     204           0 :             IncrementHoldFlowCounter();
     205           0 :             audit_list_.push_back(AuditEntry(audit_idx_, gen_id, t));
     206             :         }
     207             : 
     208           0 :         count++;
     209           0 :         audit_idx_++;
     210           0 :         if (audit_idx_ == table_entries_count_) {
     211           0 :             UpdateAgentHoldFlowCounter();
     212           0 :             audit_idx_ = 0;
     213             :         }
     214             :     }
     215           0 :     return true;
     216             : }
     217             : 
     218           0 : void KSyncMemory::GetTableSize() {
     219             :     struct nl_client *cl;
     220             :     int attr_len;
     221             :     int encode_len;
     222             : 
     223           0 :     assert((cl = nl_register_client()) != NULL);
     224           0 :     cl->cl_genl_family_id = KSyncSock::GetNetlinkFamilyId();
     225           0 :     assert(nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST) == 0);
     226           0 :     assert(nl_build_genlh(cl, SANDESH_REQUEST, 0) == 0);
     227             : 
     228           0 :     attr_len = nl_get_attr_hdr_size();
     229           0 :     encode_len = EncodeReq(cl, attr_len);
     230           0 :     nl_build_attr(cl, encode_len, NL_ATTR_VR_MESSAGE_PROTOCOL);
     231           0 :     nl_update_nlh(cl);
     232           0 :     string ksync_agent_vrouter_sock_path = KSYNC_AGENT_VROUTER_SOCK_PATH;
     233             :     ksync_agent_vrouter_sock_path =
     234           0 :     ksync_->agent()->params()->cat_is_agent_mocked()?
     235           0 :     ksync_->agent()->params()->cat_ksocketdir() + 
     236           0 :     "dpdk_netlink" : ksync_agent_vrouter_sock_path;
     237             : 
     238             : #ifdef AGENT_VROUTER_TCP
     239             :     tcp::socket socket(*(ksync_->agent()->event_manager()->io_service()));
     240             :     tcp::endpoint endpoint(ksync_->agent()->vrouter_server_ip(),
     241             :                            ksync_->agent()->vrouter_server_port());
     242             : #else
     243             :     boost::asio::local::stream_protocol::socket
     244           0 :         socket(*(ksync_->agent()->event_manager()->io_service()));
     245             :     boost::asio::local::stream_protocol::endpoint
     246           0 :         endpoint(ksync_agent_vrouter_sock_path);
     247             : #endif
     248           0 :     boost::system::error_code ec;
     249           0 :     socket.connect(endpoint, ec);
     250           0 :     if (ec) {
     251           0 :         assert(0);
     252             :     }
     253             : 
     254           0 :     socket.send(boost::asio::buffer(cl->cl_buf, cl->cl_buf_offset), 0, ec);
     255           0 :     if (ec) {
     256           0 :         assert(0);
     257             :     }
     258             : 
     259           0 :     uint32_t len_read = 0;
     260           0 :     uint32_t data_len = sizeof(struct nlmsghdr);
     261           0 :     while (len_read < data_len) {
     262           0 :         len_read = socket.read_some(boost::asio::buffer(cl->cl_buf + len_read,
     263           0 :                                                         cl->cl_buf_len), ec);
     264           0 :         if (ec) {
     265           0 :             assert(0);
     266             :         }
     267             : 
     268           0 :         if (len_read > sizeof(struct nlmsghdr)) {
     269           0 :             const struct nlmsghdr *nlh =
     270             :                 (const struct nlmsghdr *)((cl->cl_buf));
     271           0 :             data_len = nlh->nlmsg_len;
     272             :         }
     273             :     }
     274             : 
     275           0 :     KSyncSockNetlink::NetlinkDecoder(cl->cl_buf,
     276           0 :                                      KSyncSock::GetAgentSandeshContext(0));
     277           0 :     nl_free_client(cl);
     278           0 : }
     279             : 
     280           0 : void KSyncMemory::MapSharedMemory() {
     281           0 :     GetTableSize();
     282           0 :     Mmap(false, NULL, false);
     283           0 : }