Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
       3             :  */
       4             : 
       5             : #include <sys/socket.h>
       6             : #if defined(__linux__)
       7             : #include <linux/netlink.h>
       8             : #endif
       9             : #include <fcntl.h>
      10             : #include <sys/mman.h>
      11             : #include <sys/types.h>
      12             : #include <sys/ipc.h>
      13             : #include <sys/shm.h>
      14             : #include <asm/types.h>
      15             : #include <boost/asio.hpp>
      16             : 
      17             : #include <base/timer.h>
      18             : #include <base/task_trigger.h>
      19             : #include <base/address_util.h>
      20             : #include <cmn/agent_cmn.h>
      21             : #include <services/services_init.h>
      22             : #include <uve/stats_collector.h>
      23             : #include <services/icmp_error_proto.h>
      24             : #include <pkt/flow_proto.h>
      25             : #include <ksync/ksync_index.h>
      26             : #include <ksync/ksync_entry.h>
      27             : #include <ksync/ksync_object.h>
      28             : #include <ksync/ksync_netlink.h>
      29             : #include <ksync/ksync_sock.h>
      30             : #include <ksync/ksync_sock_user.h>
      31             : 
      32             : #include <vr_types.h>
      33             : #include <nl_util.h>
      34             : #include <vr_flow.h>
      35             : #include <ini_parser.h>
      36             : #include <vr_genetlink.h>
      37             : 
      38             : #include "ksync_init.h"
      39             : #include "ksync_flow_memory.h"
      40             : #include "sandesh_ksync.h"
      41             : #include "init/agent_param.h"
      42             : 
      43             : using namespace boost::asio::ip;
      44             : static const int kTestFlowTableSize = 131072 * sizeof(vr_flow_entry);
      45             : 
      46           6 : KSyncMemory::KSyncMemory(KSync *ksync, uint32_t minor_id) :
      47           6 :     ksync_(ksync),
      48           6 :     table_path_(),
      49           6 :     major_devid_(0),
      50           6 :     minor_devid_(minor_id),
      51           6 :     table_size_(0),
      52           6 :     table_entries_count_(0),
      53           6 :     audit_timer_(TimerManager::CreateTimer
      54           6 :                  (*(ksync->agent()->event_manager())->io_service(),
      55             :                   " Audit Timer",
      56             :                   ksync->agent()->task_scheduler()->GetTaskId(kTaskFlowAudit),
      57             :                   0)),
      58           6 :     audit_timeout_(0),
      59           6 :     audit_yield_(0),
      60           6 :     audit_interval_(0),
      61           6 :     audit_idx_(0),
      62          12 :     audit_list_() {
      63           6 : }
      64             : 
      65           6 : KSyncMemory::~KSyncMemory() {
      66           6 :     TimerManager::DeleteTimer(audit_timer_);
      67           6 : }
      68             : 
      69           0 : void KSyncMemory::Init() {
      70           0 :     audit_interval_ = kAuditYieldTimer;
      71           0 :     audit_timeout_ = kAuditTimeout;
      72           0 :     uint32_t table_count = table_entries_count_;
      73             :     // Compute number of entries to visit per timer interval so that complete
      74             :     // table can be visited in kAuditSweepTime
      75           0 :     uint32_t timer_per_sec = 1000 / kAuditYieldTimer;
      76           0 :     uint32_t timer_per_sweep = kAuditSweepTime * timer_per_sec;
      77           0 :     audit_yield_ = table_count / timer_per_sweep;
      78           0 :     if (audit_yield_ > kAuditYieldMax)
      79           0 :         audit_yield_ = kAuditYieldMax;
      80           0 :     if (audit_yield_ < kAuditYieldMin)
      81           0 :         audit_yield_ = kAuditYieldMin;
      82             : 
      83           0 :     audit_timer_->Start(audit_interval_,
      84             :                         boost::bind(&KSyncMemory::AuditProcess, this));
      85           0 : }
      86             : 
      87           0 : void KSyncMemory::Mmap(bool unlink_node, void *khpmem, bool kernel_mode) {
      88             :     // In case of non hugepage kernel mode,
      89             :     // Remove the existing /dev/ file first. We will add it again in vr_table_map
      90           0 :     if (!khpmem && unlink_node) {
      91           0 :         const char *error_msg = vr_table_unlink(table_path_.c_str());
      92           0 :         if (error_msg) {
      93           0 :             LOG(ERROR, "Error unmapping KSync memory: " << error_msg);
      94           0 :             assert(0);
      95             :         }
      96             :     }
      97           0 :     parse_ini_file();
      98             : 
      99             :     // Kernel hugepage present
     100           0 :     if (khpmem) {
     101           0 :         table_ = khpmem;
     102             :     } else {
     103             :         // DPDK or kernel without hugepage support
     104             :         const char *table_str;
     105           0 :         if (kernel_mode) {
     106           0 :             table_str = NULL;
     107             :         } else {
     108           0 :             table_str = table_path_.c_str();
     109             :         }
     110           0 :         const char *mmap_error_msg = vr_table_map(major_devid_, minor_devid_, table_str,
     111           0 :                                                   table_size_, &table_);
     112           0 :         if (mmap_error_msg) {
     113           0 :             LOG(ERROR, "Error mapping KSync memory. Device: " << table_path_ << "; " << mmap_error_msg);
     114           0 :             assert(0);
     115             :         }
     116           0 :         LOG(INFO, "Mem mapped dev file:" << table_path_.c_str() << " to addr:" << table_ << "\n");
     117             :     }
     118             : 
     119           0 :     table_entries_count_ = table_size_ / get_entry_size();
     120           0 :     SetTableSize();
     121           0 : }
     122             : 
     123           0 : int KSyncMemory::GetKernelTableSize() {
     124             :     struct nl_client *cl;
     125             :     int attr_len;
     126             :     int encode_len, ret;
     127             : 
     128           0 :     assert((cl = nl_register_client()) != NULL);
     129             : 
     130           0 :     assert(nl_socket(cl, AF_NETLINK, SOCK_DGRAM, NETLINK_GENERIC) > 0);
     131           0 :     assert(nl_connect(cl, 0, 0, 0) == 0);
     132             : 
     133           0 :     assert(vrouter_obtain_family_id(cl) > 0);
     134             : 
     135           0 :     assert(nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST) == 0);
     136           0 :     assert(nl_build_genlh(cl, SANDESH_REQUEST, 0) == 0);
     137             : 
     138           0 :     attr_len = nl_get_attr_hdr_size();
     139             : 
     140           0 :     encode_len = EncodeReq(cl, attr_len);
     141           0 :     nl_build_attr(cl, encode_len, NL_ATTR_VR_MESSAGE_PROTOCOL);
     142           0 :     nl_update_nlh(cl);
     143             : 
     144           0 :     if ((ret = nl_sendmsg(cl)) < 0) {
     145           0 :         LOG(DEBUG, "Error requesting  Table message. Error : " << ret);
     146           0 :         assert(0);
     147             :     }
     148             : 
     149           0 :     while ((ret = nl_recvmsg(cl)) > 0) {
     150           0 :         KSyncSockNetlink::NetlinkDecoder(cl->cl_buf,
     151           0 :                                          KSyncSock::GetAgentSandeshContext(0));
     152             :     }
     153           0 :     nl_free_client(cl);
     154           0 :     return table_size_;
     155             : }
     156             : 
     157             : // Steps to map  table entry
     158             : // In case of non huge pages
     159             : // - Query the  table parameters from kernel
     160             : // - Create device /dev/ with major-num and minor-num
     161             : // - Map device memory
     162             : // In case of huge pages
     163             : // - Just use the huge page memory which is initialized
     164             : //   and passed to this function
     165           0 : void KSyncMemory::InitMem(void *hpmem) {
     166           0 :     GetKernelTableSize();
     167           0 :     Mmap(true, hpmem, true);
     168           0 :     return;
     169             : }
     170             : 
     171           0 : void KSyncMemory::InitTest() {
     172           0 :     assert(0);
     173             : }
     174             : 
     175           0 : void KSyncMemory::Shutdown() {
     176           0 :     assert(0);
     177             : }
     178             : 
     179           0 : bool KSyncMemory::AuditProcess() {
     180             :     // Get current time
     181           0 :     uint64_t t = UTCTimestampUsec();
     182             : 
     183           0 :     while (!audit_list_.empty()) {
     184           0 :         AuditEntry list_entry = audit_list_.front();
     185             :         // audit_list_ is sorted on last time of insertion in the list
     186             :         // So, break on finding first  entry that cannot be aged
     187           0 :         if ((t - list_entry.timeout) < audit_timeout_) {
     188             :             /* Wait for audit_timeout_ to create short  for the entry */
     189           0 :             break;
     190             :         }
     191           0 :         uint32_t idx = list_entry.audit_idx;
     192           0 :         uint32_t gen_id = list_entry.audit_gen_id;
     193           0 :         audit_list_.pop_front();
     194           0 :         DecrementHoldFlowCounter();
     195           0 :         CreateProtoAuditEntry(idx, gen_id);
     196             :     }
     197             : 
     198           0 :     uint32_t count = 0;
     199             :     uint8_t gen_id;
     200           0 :     assert(audit_yield_);
     201           0 :     while (count < audit_yield_) {
     202           0 :         if (IsInactiveEntry(audit_idx_, gen_id)) {
     203           0 :             IncrementHoldFlowCounter();
     204           0 :             audit_list_.push_back(AuditEntry(audit_idx_, gen_id, t));
     205             :         }
     206             : 
     207           0 :         count++;
     208           0 :         audit_idx_++;
     209           0 :         if (audit_idx_ == table_entries_count_) {
     210           0 :             UpdateAgentHoldFlowCounter();
     211           0 :             audit_idx_ = 0;
     212             :         }
     213             :     }
     214           0 :     return true;
     215             : }
     216             : 
     217           0 : void KSyncMemory::GetTableSize() {
     218             :     struct nl_client *cl;
     219             :     int attr_len;
     220             :     int encode_len;
     221             : 
     222           0 :     assert((cl = nl_register_client()) != NULL);
     223           0 :     cl->cl_genl_family_id = KSyncSock::GetNetlinkFamilyId();
     224           0 :     assert(nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST) == 0);
     225           0 :     assert(nl_build_genlh(cl, SANDESH_REQUEST, 0) == 0);
     226             : 
     227           0 :     attr_len = nl_get_attr_hdr_size();
     228           0 :     encode_len = EncodeReq(cl, attr_len);
     229           0 :     nl_build_attr(cl, encode_len, NL_ATTR_VR_MESSAGE_PROTOCOL);
     230           0 :     nl_update_nlh(cl);
     231           0 :     string ksync_agent_vrouter_sock_path = KSYNC_AGENT_VROUTER_SOCK_PATH;
     232             :     ksync_agent_vrouter_sock_path =
     233           0 :     ksync_->agent()->params()->cat_is_agent_mocked()?
     234           0 :     ksync_->agent()->params()->cat_ksocketdir() + 
     235           0 :     "dpdk_netlink" : ksync_agent_vrouter_sock_path;
     236             : 
     237             : #ifdef AGENT_VROUTER_TCP
     238             :     tcp::socket socket(*(ksync_->agent()->event_manager()->io_service()));
     239             :     tcp::endpoint endpoint(ksync_->agent()->vrouter_server_ip(),
     240             :                            ksync_->agent()->vrouter_server_port());
     241             : #else
     242             :     boost::asio::local::stream_protocol::socket
     243           0 :         socket(*(ksync_->agent()->event_manager()->io_service()));
     244             :     boost::asio::local::stream_protocol::endpoint
     245           0 :         endpoint(ksync_agent_vrouter_sock_path);
     246             : #endif
     247           0 :     boost::system::error_code ec;
     248           0 :     socket.connect(endpoint, ec);
     249           0 :     if (ec) {
     250           0 :         assert(0);
     251             :     }
     252             : 
     253           0 :     socket.send(boost::asio::buffer(cl->cl_buf, cl->cl_buf_offset), 0, ec);
     254           0 :     if (ec) {
     255           0 :         assert(0);
     256             :     }
     257             : 
     258           0 :     uint32_t len_read = 0;
     259           0 :     uint32_t data_len = sizeof(struct nlmsghdr);
     260           0 :     while (len_read < data_len) {
     261           0 :         len_read = socket.read_some(boost::asio::buffer(cl->cl_buf + len_read,
     262           0 :                                                         cl->cl_buf_len), ec);
     263           0 :         if (ec) {
     264           0 :             assert(0);
     265             :         }
     266             : 
     267           0 :         if (len_read > sizeof(struct nlmsghdr)) {
     268           0 :             const struct nlmsghdr *nlh =
     269             :                 (const struct nlmsghdr *)((cl->cl_buf));
     270           0 :             data_len = nlh->nlmsg_len;
     271             :         }
     272             :     }
     273             : 
     274           0 :     KSyncSockNetlink::NetlinkDecoder(cl->cl_buf,
     275           0 :                                      KSyncSock::GetAgentSandeshContext(0));
     276           0 :     nl_free_client(cl);
     277           0 : }
     278             : 
     279           0 : void KSyncMemory::MapSharedMemory() {
     280           0 :     GetTableSize();
     281           0 :     Mmap(false, NULL, false);
     282           0 : }