Line data Source code
1 : /*
2 : * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
3 : * Copyright Contributors to the OpenSDN project
4 : */
5 : #ifndef __AGENT_FLOW_TABLE_MGMT_H__
6 : #define __AGENT_FLOW_TABLE_MGMT_H__
7 :
8 : #include <boost/scoped_ptr.hpp>
9 : #include "pkt/flow_table.h"
10 : #include <pkt/flow_mgmt/flow_mgmt_dbclient.h>
11 : #include <pkt/flow_mgmt/flow_mgmt_tree.h>
12 : #include "pkt/flow_event.h"
13 :
14 : ////////////////////////////////////////////////////////////////////////////
15 : // Flow Management module is responsible to keep flow action in-sync with
16 : // changes to operational-db changes.
17 : //
18 : // Flow Management module tracks following,
19 : // 1. Changes to fields of interest in the operational-db
20 : // 2. Flows dependenent on operational-db entries
21 : //
22 : // On any change to an operational-db, it will ensure the dependent flows are
23 : // revaluated to keep the flow action in-sync with latest operational state
24 : //
25 : // A flow can depend on following operational-entires,
26 : // 1. Interface : Flow depends on the SG-Id of interface
27 : // 2. VN : Flow depends on policy of VN
28 : // 3. ACL : Flow depends on ACL Entries in ACL
29 : // 4. Route : Flow depends on the SG-Id list in the route for both source
30 : // and destination addresses
31 : // 5. NextHop : Flow depends on NH for RPF checks and for ECMP Component index
32 : //
33 : // An operational entry can potentially have many flows dependent on it. On
34 : // change to an operational entry if flows are processed inline, it can lead to
35 : // significant latency. The flow management module runs from a work-queue
36 : // (flow-manager queue) to avoid the latency.
37 : //
38 : // The Flow Management module is responsible only to track the dependency
39 : // between flows and operational entries. It does not *own* the flow entries.
40 : // When flow management module identifies that a flow has to be revaluated
41 : // it will invoke flow-table APIs.
42 : //
43 : // There are two sub-modules,
44 : // - Flow Management DBClient
45 : // This module registers to DB notification for all tables of interest. It
46 : // enqueues events for flow revaluation in response to DBTable changes
47 : // - Flow Management Tree module
48 : // This module maintains different data-structures to track flows affected
49 : // by change to a DBEntry. Its also responsible to trigger revaluation of
50 : // flows on change to a DBEntry
51 : //
52 : // There are 2 work-queues defined for flow-management,
53 : // - Flow Management Request
54 : // All request events to flow-management module are enqueued thru this.
55 : // The Flow Management Tree module acts on events on this queue.
56 : // * Add of FLow
57 : // * Delete of Flow
58 : // * Add of DBEntry
59 : // * Change of DBEntry
60 : // * Delete of DBEntry
61 : // * Export of Flow
62 : //
63 : // - Flow Events
64 : // Flow Management Tree module may generate events in response to
65 : // requests. The response events are enqueued here. Example events are,
66 : // * Flow revaluation in response to DBEntry change
67 : // * Flow revaluation in response to DBEntry Add/Delete
68 : // * Flow deletion in response to DBEntry delete
69 : //
70 : // Workflow for flow manager module is given below,
71 : // 1. Flow Table module will enqueue message to Flow Management queue on
72 : // add/delete/change of a flow. On Flow delete event, Flow Table module will
73 : // will also enqueue export of the flow.
74 : // 2. Flow stats collection module will enqueue message to Flow Management
75 : // queue on export of flow
76 : // 3. Flow Management module builds the following tracking information
77 : // - Operational entry to list of dependent flows
78 : // - Flow entry to list of operational-entries it is dependent on
79 : // 4. DBClient module registers to DBTables of interest and tracks changes to
80 : // operational-db entries
81 : // 5. Flow Table module will enqueue a message to Flow Management queue on
82 : // add/delete/change of operational entries
83 : // 6. The action in flow-management module for operational entry events will
84 : // depend on the operational entry type
85 : //
86 : // VN Add/Change : Revaluate flows for change in policy
87 : // VN Delete : Delete dependent flows
88 : // VMI Add/Change : Revaluate flows for change in SG entry in VMI
89 : // Interface Delete : Delete dependent flows
90 : // NH Add/Change : Revaluate flows for change in RPF-NH
91 : // NH Delete : Delete dependent flows
92 : // ACL Add/Change : Revaluate flows for change in ACL rules
93 : // ACL Delete : Delete dependent flows
94 : // Bridge Add/Change : Flow entries are revaluated for change in SG
95 : // Bridge Delete : Delete dependent flows
96 : // Inet Route Add : Add of an inet route can potentially alter the route
97 : // used for a route. It can also potentially alter
98 : // other attributes such as floating-ip used etc...
99 : // So, all flows dependent on the "covering route" are
100 : // enqueued for revluation by PktFlowInfo module
101 : // Inet Route Change : Revluate for for change in RPF-NH, VN and SG
102 : // Inet Route Delete : Flows depending on this route will start using the
103 : // covering route. It can also potentially alter
104 : // other attributes such as floating-ip used etc...
105 : // So, all flows dependent on the "covering route" are
106 : // enqueued for revluation by PktFlowInfo module
107 : //
108 : // Concurrency and references
109 : // The Flow Management module can often lead to compute spikes. To ensure this
110 : // doesnt affect flow setup rates, it is preferable to run flow-management
111 : // module in parallel to both DBTable and Flow setup/teardown task.
112 : //
113 : // Since flow-management module runs in parallel to DBTable/Flow tasks, we must
114 : // ensure that flows and the operational dbentries are not deleted till
115 : // flow-management module references are gone. This is achieved by following,
116 : //
117 : // Flow reference
118 : // --------------
119 : // 1. All message between flow-management and flow-table/flow-stats module will
120 : // hold object references. This will ensure ref-count for object dont drop
121 : // till messages are processed.
122 : // 2. Each flow seen by flow-management will hold a reference to FlowEntryInfo
123 : // in FlowEntry as flow_mgmt_info_
124 : //
125 : // Per FlowEntry mutex is used to synchronize access to same Flow between
126 : // FlowTable and Flow Management module
127 : //
128 : // DBEntry reference
129 : // -----------------
130 : // Flow Management module will rely on DBState set by DBClient module to
131 : // ensure a DBEntry is not deleted till all its flows are deleted.
132 : //
133 : // Flow Management DBClient Flow Management module
134 : //
135 : // 1. On DBEntry deletion, enqueue a
136 : // message to Flow Management module
137 : // 2. Find all dependent flows and
138 : // enqueue flow-delete request for
139 : // all flows. Mark the entry as
140 : // deleted
141 : // 3. Delete the flow and enqueue a
142 : // flow-deleted message to
143 : // flow-management module
144 : // 4. Find all operational entries the
145 : // flow is dependent on and remove
146 : // the flow from their dependent tree.
147 : // If an operational entry is deleted
148 : // and there are no more dependent
149 : // flows, enqueue a message to delete
150 : // the operational entry
151 : // 5. Remove DBState for the DBEntry
152 : //
153 : // RouteTable reference
154 : // -----------------
155 : // The Inet and Bridge route tables should not be deleted till all flows using
156 : // them are deleted. Flow Management module holds Lifetime reference to ensure
157 : // DBTables are deleted only after all its flows are deleted
158 : //
159 : // VRF Reference
160 : // -------------
161 : // When a VRF add is seen, the DBClient sub-module will register for Inet and
162 : // Bridge route tables for the VRF. The unregister for these tables should
163 : // happen only after all flows on them are deleted. The Flow Management module
164 : // will enqueue delete of VRF only after flow entries in all Inet and Bridge
165 : // routes are deleted
166 : //
167 : // Additional functionality:
168 : // -------------------------
169 : // A few more additional functionality is pushed to flow-management module
170 : // 1. Logging of flows
171 : // 2. Flow UVE
172 : // 3. Tracking per VN flows
173 : //
174 : // Duplicate Deletes:
175 : // -----------------
176 : // Consider following sequence of events,
177 : // 1. Add ACL
178 : // 2. Delete ACL
179 : // 3. Delete ACL
180 : //
181 : // Correspondingly, 3 events are enqueued to Flow Management Request queue.
182 : // When event (2) is processed, if ACL does not have any more flows, it will
183 : // result in removing DBState for the ACL and eventual free of ACL before
184 : // event (3) is processed. When event (3) is being processed, it may refer to
185 : // a free'd DBEntry
186 : //
187 : // We use a gen-id to handle this scenario.The gen-id is incremented on every
188 : // Delete notification received for DBEntry. When DELETE event is enqueued, it
189 : // will also carry the gen-id field. The gen-id is also carried in the
190 : // response events generated.
191 : //
192 : // When Flow Management Dbclient module receives FREE_DBENTRY event, it will
193 : // ignore the message if gen-id does not match with latest value.
194 : ////////////////////////////////////////////////////////////////////////////
195 :
196 : ////////////////////////////////////////////////////////////////////////////
197 : // Flow Management module maintains following data structures
198 : //
199 : // - FlowEntryTree : Tree of all flow entries
200 : // FlowEntryPtr : Key for the tree. Holds reference to FlowEntry
201 : // FlowEntryInfo : Data for the tree. Contains tree of DBEntries the flow is
202 : // dependent on.
203 : //
204 : // - FlowMgmtTree : Per operational entry tree. Tracks flow entries dependent
205 : // on the operational entry
206 : //
207 : // An entry into the tree can be added when,
208 : // 1. Flow is added/changed and it refers to the DBEntry
209 : // or
210 : // 2. DBEntry add/change message is got from DBClient
211 : //
212 : // Entry from the tree is removed,
213 : // 1. All flows dependent on the entry are deleted
214 : // AND
215 : // 2. DBEntry delete message is got from DBClient
216 : //
217 : // FlowEntryKey : Key for the tree. Contains DBEntry pointer as key
218 : // FlowMgmtEntry : Data fot the tree. Maintains intrusive-list of
219 : // FlowMgmtKeyNodes(which contains references to Flow entries)
220 : // dependent on the DBEntry
221 : //
222 : // - AclFlowMgmtTree : FlowMgmtTree for ACL
223 : // - InterfaceFlowMgmtTree : FlowMgmtTree for VM-Interfaces
224 : // - VnFlowMgmtTree : FlowMgmtTree for VN
225 : // - InetRouteFlowMgmtTree : FlowMgmtTree for IPv4 routes
226 : // - InetRouteFlowMgmtTree : FlowMgmtTree for IPv6 routes
227 : // - BridgeRouteFlowMgmtTree: FlowMgmtTree for Bridge routes
228 : // - NhFlowMgmtTree : FlowMgmtTree for NH
229 : // - VrfFlowMgmtTree : FlowMgmtTree for VRF. It doesnt track the flows
230 : // for the VRF. But is used to ensure VRF entry is
231 : // not deleted till all route-entries are freed and
232 : // DELETE event for VRF is processed
233 : // - BgpAsAServiceFlowMgmtTree : FlowMgmtTree per control-node. This is
234 : // maintained per control node because VMI can
235 : // establish a bgp peer session with each control
236 : // node.
237 : ////////////////////////////////////////////////////////////////////////////
238 :
239 : class FlowMgmtManager {
240 : public:
241 : typedef boost::shared_ptr<FlowMgmtRequest> FlowMgmtRequestPtr;
242 : typedef WorkQueue<FlowMgmtRequestPtr> FlowMgmtQueue;
243 :
244 : // Comparator for FlowEntryPtr
245 : struct FlowEntryRefCmp {
246 : bool operator()(const FlowEntryPtr &l, const FlowEntryPtr &r) {
247 : FlowEntry *lhs = l.get();
248 : FlowEntry *rhs = r.get();
249 :
250 : return (lhs < rhs);
251 : }
252 : };
253 :
254 : // We want flow to be valid till Flow Management task is complete. So,
255 : // use FlowEntryPtr as key and hold reference to flow till we are done
256 : typedef std::map<FlowEntryPtr, FlowEntryInfo, FlowEntryRefCmp>
257 : FlowEntryTree;
258 :
259 : FlowMgmtManager(Agent *agent, uint16_t table_index);
260 8 : virtual ~FlowMgmtManager() { }
261 :
262 : void Init();
263 : void Shutdown();
264 : static void InitLogQueue(Agent *agent);
265 : static void ShutdownLogQueue();
266 : static void LogFlowUnlocked(FlowEntry *flow, const std::string &op);
267 : static bool LogHandler(FlowMgmtRequestPtr req);
268 :
269 : bool RequestHandler(FlowMgmtRequestPtr req);
270 : bool DBRequestHandler(FlowMgmtRequestPtr req);
271 : static bool ProcessEvent(FlowMgmtRequest *req, FlowMgmtKey *key,
272 : FlowMgmtTree *tree);
273 :
274 : bool DBRequestHandler(FlowMgmtRequest *req, const DBEntry *entry);
275 : bool BgpAsAServiceRequestHandler(FlowMgmtRequest *req);
276 : bool DbClientHandler(const DBEntry *entry);
277 : void EnqueueFlowEvent(FlowEvent *event);
278 : void NonOperEntryEvent(FlowEvent::Event event, FlowEntry *flow);
279 : void DBEntryEvent(FlowEvent::Event event, FlowMgmtKey *key,
280 : FlowEntry *flow);
281 : void FreeDBEntryEvent(FlowEvent::Event event, FlowMgmtKey *key,
282 : uint32_t gen_id);
283 :
284 1282 : Agent *agent() const { return agent_; }
285 : uint16_t table_index() const { return table_index_; }
286 : void AddEvent(FlowEntry *low);
287 : void DeleteEvent(FlowEntry *flow, const RevFlowDepParams ¶ms);
288 : void FlowStatsUpdateEvent(FlowEntry *flow, uint32_t bytes, uint32_t packets,
289 : uint32_t oflow_bytes,
290 : const boost::uuids::uuid &u);
291 : void AddDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
292 : void ChangeDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
293 : void DeleteDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
294 : void RouteNHChangeEvent(const DBEntry *entry, uint32_t gen_id);
295 : void RetryVrfDeleteEvent(const VrfEntry *vrf);
296 : void RetryVrfDelete(uint32_t vrf_id);
297 : // Dummy event used for testing
298 : void DummyEvent();
299 :
300 : void VnFlowCounters(const VnEntry *vn,
301 : uint32_t *ingress_flow_count,
302 : uint32_t *egress_flow_count);
303 : void InterfaceFlowCount(const Interface *itf, uint64_t *created,
304 : uint64_t *aged, uint32_t *active_flows);
305 : bool HasVrfFlows(uint32_t vrf);
306 :
307 3494 : FlowMgmtDbClient *flow_mgmt_dbclient() const {
308 3494 : return flow_mgmt_dbclient_.get();
309 : }
310 :
311 5 : const FlowMgmtQueue *request_queue() const { return &request_queue_; }
312 : const FlowMgmtQueue *log_queue() const { return log_queue_; }
313 1 : void DisableWorkQueue(bool disable) { request_queue_.set_disable(disable); }
314 : void BgpAsAServiceNotify(const boost::uuids::uuid &vm_uuid,
315 : uint32_t source_port);
316 : void BgpAsAServiceHealthCheckNotify(const boost::uuids::uuid &vm_uuid,
317 : uint32_t source_port,
318 : const boost::uuids::uuid &hc_uuid,
319 : bool add);
320 : void EnqueueUveAddEvent(const FlowEntry *flow) const;
321 : void EnqueueUveDeleteEvent(const FlowEntry *flow) const;
322 :
323 : void FlowUpdateQueueDisable(bool val);
324 : size_t FlowUpdateQueueLength();
325 : size_t FlowDBQueueLength();
326 144 : InetRouteFlowMgmtTree* ip4_route_flow_mgmt_tree() {
327 144 : return &ip4_route_flow_mgmt_tree_;
328 : }
329 :
330 : BgpAsAServiceFlowMgmtKey *FindBgpAsAServiceInfo(
331 : FlowEntry *flow,
332 : BgpAsAServiceFlowMgmtKey &key);
333 :
334 : private:
335 : // Handle Add/Change of a flow. Builds FlowMgmtKeyTree for all objects
336 : void AddFlow(FlowEntryPtr &flow);
337 : // Handle Delete of a flow. Updates FlowMgmtKeyTree for all objects
338 : void DeleteFlow(FlowEntryPtr &flow, const RevFlowDepParams &p);
339 : void UpdateFlowStats(FlowEntryPtr &flow, uint32_t bytes, uint32_t packets,
340 : uint32_t oflow_bytes, const boost::uuids::uuid &u);
341 :
342 : // Add a FlowMgmtKey into the FlowMgmtKeyTree for an object
343 : // The FlowMgmtKeyTree for object is passed as argument
344 : void AddFlowMgmtKey(FlowEntry *flow, FlowEntryInfo *info,
345 : FlowMgmtKey *key, FlowMgmtKey *old_key);
346 : // Delete a FlowMgmtKey from FlowMgmtKeyTree for an object
347 : // The FlowMgmtKeyTree for object is passed as argument
348 : void DeleteFlowMgmtKey(FlowEntry *flow, FlowEntryInfo *info,
349 : FlowMgmtKey *key, FlowMgmtKeyNode *node);
350 : FlowEntryInfo *FindFlowEntryInfo(const FlowEntryPtr &flow);
351 : FlowEntryInfo *LocateFlowEntryInfo(FlowEntryPtr &flow);
352 : void DeleteFlowEntryInfo(FlowEntryPtr &flow);
353 : void MakeFlowMgmtKeyTree(FlowEntry *flow, FlowMgmtKeyTree *tree);
354 : void SetAceSandeshData(const AclDBEntry *acl, AclFlowCountResp &data,
355 : const std::string &ace_id);
356 : void SetAclFlowSandeshData(const AclDBEntry *acl, AclFlowResp &data,
357 : const int last_count);
358 : void ControllerNotify(uint8_t index);
359 :
360 : Agent *agent_;
361 : uint16_t table_index_;
362 : AclFlowMgmtTree acl_flow_mgmt_tree_;
363 : InterfaceFlowMgmtTree interface_flow_mgmt_tree_;
364 : VnFlowMgmtTree vn_flow_mgmt_tree_;
365 : InetRouteFlowMgmtTree ip4_route_flow_mgmt_tree_;
366 : InetRouteFlowMgmtTree ip6_route_flow_mgmt_tree_;
367 : BridgeRouteFlowMgmtTree bridge_route_flow_mgmt_tree_;
368 : VrfFlowMgmtTree vrf_flow_mgmt_tree_;
369 : NhFlowMgmtTree nh_flow_mgmt_tree_;
370 : boost::scoped_ptr<BgpAsAServiceFlowMgmtTree> bgp_as_a_service_flow_mgmt_tree_[MAX_XMPP_SERVERS];
371 : std::unique_ptr<FlowMgmtDbClient> flow_mgmt_dbclient_;
372 : FlowMgmtQueue request_queue_;
373 : FlowMgmtQueue db_event_queue_;
374 : static FlowMgmtQueue *log_queue_;
375 : DISALLOW_COPY_AND_ASSIGN(FlowMgmtManager);
376 : };
377 :
378 : #define FLOW_TRACE(obj, ...)\
379 : do {\
380 : Flow##obj::TraceMsg(FlowTraceBuf, __FILE__, __LINE__, ##__VA_ARGS__);\
381 : } while (false)
382 :
383 : #endif // __AGENT_FLOW_TABLE_MGMT_H__
|
