DPDK  23.11.0
rte_graph_worker_common.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2020 Marvell International Ltd.
3  */
4 
5 #ifndef _RTE_GRAPH_WORKER_COMMON_H_
6 #define _RTE_GRAPH_WORKER_COMMON_H_
7 
15 #include <rte_common.h>
16 #include <rte_cycles.h>
17 #include <rte_prefetch.h>
18 #include <rte_memcpy.h>
19 #include <rte_memory.h>
20 
21 #include "rte_graph.h"
22 
23 #ifdef __cplusplus
24 extern "C" {
25 #endif
26 
28 /* When adding a new graph model entry, update rte_graph_model_is_valid() implementation. */
29 #define RTE_GRAPH_MODEL_RTC 0
30 #define RTE_GRAPH_MODEL_MCORE_DISPATCH 1
32 #define RTE_GRAPH_MODEL_DEFAULT RTE_GRAPH_MODEL_RTC
39 SLIST_HEAD(rte_graph_rq_head, rte_graph);
40 
46 struct rte_graph {
47  /* Fast path area. */
48  uint32_t tail;
49  uint32_t head;
50  uint32_t cir_mask;
51  rte_node_t nb_nodes;
52  rte_graph_off_t *cir_start;
53  rte_graph_off_t nodes_start;
54  uint8_t model;
55  uint8_t reserved1;
56  uint16_t reserved2;
57  union {
58  /* Fast schedule area for mcore dispatch model */
59  struct {
60  struct rte_graph_rq_head *rq __rte_cache_aligned; /* The run-queue */
61  struct rte_graph_rq_head rq_head; /* The head for run-queue list */
62 
63  unsigned int lcore_id;
64  struct rte_ring *wq;
65  struct rte_mempool *mp;
66  } dispatch;
67  };
68  SLIST_ENTRY(rte_graph) next; /* The next for rte_graph list */
69  /* End of Fast path area.*/
70  rte_graph_t id;
71  int socket;
72  char name[RTE_GRAPH_NAMESIZE];
73  bool pcap_enable;
75  uint64_t nb_pkt_captured;
77  uint64_t nb_pkt_to_capture;
78  char pcap_filename[RTE_GRAPH_PCAP_FILE_SZ];
79  uint64_t fence;
81 
87 struct rte_node {
88  /* Slow path area */
89  uint64_t fence;
90  rte_graph_off_t next;
91  rte_node_t id;
92  rte_node_t parent_id;
93  rte_edge_t nb_edges;
94  uint32_t realloc_count;
96  char parent[RTE_NODE_NAMESIZE];
97  char name[RTE_NODE_NAMESIZE];
100  rte_node_process_t original_process;
101 
102  union {
103  /* Fast schedule area for mcore dispatch model */
104  struct {
105  unsigned int lcore_id;
106  uint64_t total_sched_objs;
107  uint64_t total_sched_fail;
108  } dispatch;
109  };
110  /* Fast path area */
111 #define RTE_NODE_CTX_SZ 16
112  uint8_t ctx[RTE_NODE_CTX_SZ] __rte_cache_aligned;
113  uint16_t size;
114  uint16_t idx;
115  rte_graph_off_t off;
116  uint64_t total_cycles;
117  uint64_t total_calls;
118  uint64_t total_objs;
119  union {
120  void **objs;
121  uint64_t objs_u64;
122  };
123  union {
124  rte_node_process_t process;
125  uint64_t process_u64;
126  };
127  struct rte_node *nodes[] __rte_cache_min_aligned;
129 
142 void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
143 
158 void __rte_node_stream_alloc_size(struct rte_graph *graph,
159  struct rte_node *node, uint16_t req_size);
160 
161 /* Fast path helper functions */
162 
173 static __rte_always_inline void
174 __rte_node_process(struct rte_graph *graph, struct rte_node *node)
175 {
176  uint64_t start;
177  uint16_t rc;
178  void **objs;
179 
180  RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
181  objs = node->objs;
182  rte_prefetch0(objs);
183 
185  start = rte_rdtsc();
186  rc = node->process(graph, node, objs, node->idx);
187  node->total_cycles += rte_rdtsc() - start;
188  node->total_calls++;
189  node->total_objs += rc;
190  } else {
191  node->process(graph, node, objs, node->idx);
192  }
193  node->idx = 0;
194 }
195 
206 static __rte_always_inline void
207 __rte_node_enqueue_tail_update(struct rte_graph *graph, struct rte_node *node)
208 {
209  uint32_t tail;
210 
211  tail = graph->tail;
212  graph->cir_start[tail++] = node->off;
213  graph->tail = tail & graph->cir_mask;
214 }
215 
233 static __rte_always_inline void
234 __rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,
235  const uint16_t idx, const uint16_t space)
236 {
237 
238  /* Add to the pending stream list if the node is new */
239  if (idx == 0)
240  __rte_node_enqueue_tail_update(graph, node);
241 
242  if (unlikely(node->size < (idx + space)))
243  __rte_node_stream_alloc_size(graph, node, node->size + space);
244 }
245 
259 static __rte_always_inline struct rte_node *
260 __rte_node_next_node_get(struct rte_node *node, rte_edge_t next)
261 {
262  RTE_ASSERT(next < node->nb_edges);
263  RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
264  node = node->nodes[next];
265  RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
266 
267  return node;
268 }
269 
285 static inline void
286 rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,
287  rte_edge_t next, void **objs, uint16_t nb_objs)
288 {
289  node = __rte_node_next_node_get(node, next);
290  const uint16_t idx = node->idx;
291 
292  __rte_node_enqueue_prologue(graph, node, idx, nb_objs);
293 
294  rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));
295  node->idx = idx + nb_objs;
296 }
297 
311 static inline void
312 rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,
313  rte_edge_t next, void *obj)
314 {
315  node = __rte_node_next_node_get(node, next);
316  uint16_t idx = node->idx;
317 
318  __rte_node_enqueue_prologue(graph, node, idx, 1);
319 
320  node->objs[idx++] = obj;
321  node->idx = idx;
322 }
323 
340 static inline void
341 rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,
342  rte_edge_t next, void *obj0, void *obj1)
343 {
344  node = __rte_node_next_node_get(node, next);
345  uint16_t idx = node->idx;
346 
347  __rte_node_enqueue_prologue(graph, node, idx, 2);
348 
349  node->objs[idx++] = obj0;
350  node->objs[idx++] = obj1;
351  node->idx = idx;
352 }
353 
374 static inline void
375 rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,
376  rte_edge_t next, void *obj0, void *obj1, void *obj2,
377  void *obj3)
378 {
379  node = __rte_node_next_node_get(node, next);
380  uint16_t idx = node->idx;
381 
382  __rte_node_enqueue_prologue(graph, node, idx, 4);
383 
384  node->objs[idx++] = obj0;
385  node->objs[idx++] = obj1;
386  node->objs[idx++] = obj2;
387  node->objs[idx++] = obj3;
388  node->idx = idx;
389 }
390 
407 static inline void
408 rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,
409  rte_edge_t *nexts, void **objs, uint16_t nb_objs)
410 {
411  uint16_t i;
412 
413  for (i = 0; i < nb_objs; i++)
414  rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);
415 }
416 
436 static inline void **
437 rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,
438  rte_edge_t next, uint16_t nb_objs)
439 {
440  node = __rte_node_next_node_get(node, next);
441  const uint16_t idx = node->idx;
442  uint16_t free_space = node->size - idx;
443 
444  if (unlikely(free_space < nb_objs))
445  __rte_node_stream_alloc_size(graph, node, node->size + nb_objs);
446 
447  return &node->objs[idx];
448 }
449 
466 static inline void
467 rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,
468  rte_edge_t next, uint16_t idx)
469 {
470  if (unlikely(!idx))
471  return;
472 
473  node = __rte_node_next_node_get(node, next);
474  if (node->idx == 0)
475  __rte_node_enqueue_tail_update(graph, node);
476 
477  node->idx += idx;
478 }
479 
494 static inline void
495 rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src,
496  rte_edge_t next)
497 {
498  struct rte_node *dst = __rte_node_next_node_get(src, next);
499 
500  /* Let swap the pointers if dst don't have valid objs */
501  if (likely(dst->idx == 0)) {
502  void **dobjs = dst->objs;
503  uint16_t dsz = dst->size;
504  dst->objs = src->objs;
505  dst->size = src->size;
506  src->objs = dobjs;
507  src->size = dsz;
508  dst->idx = src->idx;
509  __rte_node_enqueue_tail_update(graph, dst);
510  } else { /* Move the objects from src node to dst node */
511  rte_node_enqueue(graph, src, next, src->objs, src->idx);
512  }
513 }
514 
524 bool
526 
537 int rte_graph_worker_model_set(uint8_t model);
538 
551 uint8_t rte_graph_worker_model_get(struct rte_graph *graph);
552 
565 static __rte_always_inline
566 uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
567 {
568  return graph->model;
569 }
570 
571 #ifdef __cplusplus
572 }
573 #endif
574 
575 #endif /* _RTE_GRAPH_WORKER_COIMMON_H_ */
#define likely(x)
#define unlikely(x)
#define __rte_cache_aligned
Definition: rte_common.h:524
#define __rte_cache_min_aligned
Definition: rte_common.h:528
#define __rte_always_inline
Definition: rte_common.h:331
uint16_t(* rte_node_process_t)(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs)
Definition: rte_graph.h:93
uint32_t rte_node_t
Definition: rte_graph.h:40
#define RTE_GRAPH_FENCE
Definition: rte_graph.h:37
uint16_t rte_edge_t
Definition: rte_graph.h:41
#define RTE_NODE_NAMESIZE
Definition: rte_graph.h:31
#define RTE_GRAPH_PCAP_FILE_SZ
Definition: rte_graph.h:32
uint32_t rte_graph_off_t
Definition: rte_graph.h:39
uint16_t rte_graph_t
Definition: rte_graph.h:42
#define RTE_GRAPH_NAMESIZE
Definition: rte_graph.h:30
static __rte_always_inline int rte_graph_has_stats_feature(void)
Definition: rte_graph.h:686
static __rte_always_inline uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
static void rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1, void *obj2, void *obj3)
bool rte_graph_model_is_valid(uint8_t model)
static void rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t idx)
static void rte_node_enqueue(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void **objs, uint16_t nb_objs)
static void rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src, rte_edge_t next)
int rte_graph_worker_model_set(uint8_t model)
static void rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node, rte_edge_t *nexts, void **objs, uint16_t nb_objs)
static void rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj)
static void rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1)
uint8_t rte_graph_worker_model_get(struct rte_graph *graph)
static void ** rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t nb_objs)
static void * rte_memcpy(void *dst, const void *src, size_t n)
static void rte_prefetch0(const volatile void *p)
char name[RTE_MEMPOOL_NAMESIZE]
Definition: rte_mempool.h:221