DPDK  23.11.0
rte_dmadev.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2021 HiSilicon Limited
3  * Copyright(c) 2021 Intel Corporation
4  * Copyright(c) 2021 Marvell International Ltd
5  * Copyright(c) 2021 SmartShare Systems
6  */
7 
8 #ifndef RTE_DMADEV_H
9 #define RTE_DMADEV_H
10 
147 #include <stdint.h>
148 
149 #include <rte_bitops.h>
150 #include <rte_common.h>
151 
152 #ifdef __cplusplus
153 extern "C" {
154 #endif
155 
157 #define RTE_DMADEV_DEFAULT_MAX 64
158 
171 int rte_dma_dev_max(size_t dev_max);
172 
183 int rte_dma_get_dev_id_by_name(const char *name);
184 
194 bool rte_dma_is_valid(int16_t dev_id);
195 
203 uint16_t rte_dma_count_avail(void);
204 
213 int16_t rte_dma_next_dev(int16_t start_dev_id);
214 
216 #define RTE_DMA_FOREACH_DEV(p) \
217  for (p = rte_dma_next_dev(0); \
218  p != -1; \
219  p = rte_dma_next_dev(p + 1))
220 
221 
226 #define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0)
228 #define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1)
230 #define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2)
232 #define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3)
239 #define RTE_DMA_CAPA_SVA RTE_BIT64(4)
245 #define RTE_DMA_CAPA_SILENT RTE_BIT64(5)
253 #define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)
260 #define RTE_DMA_CAPA_M2D_AUTO_FREE RTE_BIT64(7)
261 
266 #define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32)
268 #define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33)
270 #define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34)
278 struct rte_dma_info {
279  const char *dev_name;
281  uint64_t dev_capa;
283  uint16_t max_vchans;
285  uint16_t max_desc;
287  uint16_t min_desc;
295  uint16_t max_sges;
297  int16_t numa_node;
299  uint16_t nb_vchans;
300 };
301 
314 int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);
315 
321 struct rte_dma_conf {
326  uint16_t nb_vchans;
335 };
336 
353 int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);
354 
367 int rte_dma_start(int16_t dev_id);
368 
380 int rte_dma_stop(int16_t dev_id);
381 
393 int rte_dma_close(int16_t dev_id);
394 
433 };
434 
441  RTE_DMA_PORT_NONE,
443 };
444 
457  union {
506  __extension__
507  struct {
508  uint64_t coreid : 4;
509  uint64_t pfid : 8;
510  uint64_t vfen : 1;
511  uint64_t vfid : 16;
513  uint64_t pasid : 20;
515  uint64_t attr : 3;
517  uint64_t ph : 2;
519  uint64_t st : 16;
520  } pcie;
521  };
522  uint64_t reserved[2];
523 };
524 
529  union {
530  struct {
538  struct rte_mempool *pool;
539  } m2d;
540  };
542  uint64_t reserved[2];
543 };
544 
557  uint16_t nb_desc;
582 };
583 
599 int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
600  const struct rte_dma_vchan_conf *conf);
601 
609  uint64_t submitted;
613  uint64_t completed;
615  uint64_t errors;
616 };
617 
624 #define RTE_DMA_ALL_VCHAN 0xFFFFu
625 
641 int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
642  struct rte_dma_stats *stats);
643 
656 int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);
657 
668 };
669 
685 int
686 rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);
687 
699 int rte_dma_dump(int16_t dev_id, FILE *f);
700 
763 };
764 
770 struct rte_dma_sge {
772  uint32_t length;
773 };
774 
775 #include "rte_dmadev_core.h"
776 
788 #define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0)
793 #define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1)
798 #define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2)
805 #define RTE_DMA_OP_FLAG_AUTO_FREE RTE_BIT64(3)
834 static inline int
835 rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
836  uint32_t length, uint64_t flags)
837 {
838  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
839 
840 #ifdef RTE_DMADEV_DEBUG
841  if (!rte_dma_is_valid(dev_id) || length == 0)
842  return -EINVAL;
843  if (*obj->copy == NULL)
844  return -ENOTSUP;
845 #endif
846 
847  return (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);
848 }
849 
880 static inline int
881 rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
882  struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
883  uint64_t flags)
884 {
885  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
886 
887 #ifdef RTE_DMADEV_DEBUG
888  if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
889  nb_src == 0 || nb_dst == 0)
890  return -EINVAL;
891  if (*obj->copy_sg == NULL)
892  return -ENOTSUP;
893 #endif
894 
895  return (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,
896  nb_dst, flags);
897 }
898 
925 static inline int
926 rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
927  rte_iova_t dst, uint32_t length, uint64_t flags)
928 {
929  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
930 
931 #ifdef RTE_DMADEV_DEBUG
932  if (!rte_dma_is_valid(dev_id) || length == 0)
933  return -EINVAL;
934  if (*obj->fill == NULL)
935  return -ENOTSUP;
936 #endif
937 
938  return (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,
939  flags);
940 }
941 
956 static inline int
957 rte_dma_submit(int16_t dev_id, uint16_t vchan)
958 {
959  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
960 
961 #ifdef RTE_DMADEV_DEBUG
962  if (!rte_dma_is_valid(dev_id))
963  return -EINVAL;
964  if (*obj->submit == NULL)
965  return -ENOTSUP;
966 #endif
967 
968  return (*obj->submit)(obj->dev_private, vchan);
969 }
970 
993 static inline uint16_t
994 rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
995  uint16_t *last_idx, bool *has_error)
996 {
997  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
998  uint16_t idx;
999  bool err;
1000 
1001 #ifdef RTE_DMADEV_DEBUG
1002  if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
1003  return 0;
1004  if (*obj->completed == NULL)
1005  return 0;
1006 #endif
1007 
1008  /* Ensure the pointer values are non-null to simplify drivers.
1009  * In most cases these should be compile time evaluated, since this is
1010  * an inline function.
1011  * - If NULL is explicitly passed as parameter, then compiler knows the
1012  * value is NULL
1013  * - If address of local variable is passed as parameter, then compiler
1014  * can know it's non-NULL.
1015  */
1016  if (last_idx == NULL)
1017  last_idx = &idx;
1018  if (has_error == NULL)
1019  has_error = &err;
1020 
1021  *has_error = false;
1022  return (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,
1023  has_error);
1024 }
1025 
1052 static inline uint16_t
1053 rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
1054  const uint16_t nb_cpls, uint16_t *last_idx,
1055  enum rte_dma_status_code *status)
1056 {
1057  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1058  uint16_t idx;
1059 
1060 #ifdef RTE_DMADEV_DEBUG
1061  if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
1062  return 0;
1063  if (*obj->completed_status == NULL)
1064  return 0;
1065 #endif
1066 
1067  if (last_idx == NULL)
1068  last_idx = &idx;
1069 
1070  return (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,
1071  last_idx, status);
1072 }
1073 
1086 static inline uint16_t
1087 rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
1088 {
1089  struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1090 
1091 #ifdef RTE_DMADEV_DEBUG
1092  if (!rte_dma_is_valid(dev_id))
1093  return 0;
1094  if (*obj->burst_capacity == NULL)
1095  return 0;
1096 #endif
1097  return (*obj->burst_capacity)(obj->dev_private, vchan);
1098 }
1099 
1100 #ifdef __cplusplus
1101 }
1102 #endif
1103 
1104 #endif /* RTE_DMADEV_H */
uint64_t rte_iova_t
Definition: rte_common.h:556
rte_dma_direction
Definition: rte_dmadev.h:400
@ RTE_DMA_DIR_MEM_TO_DEV
Definition: rte_dmadev.h:414
@ RTE_DMA_DIR_DEV_TO_MEM
Definition: rte_dmadev.h:423
@ RTE_DMA_DIR_MEM_TO_MEM
Definition: rte_dmadev.h:405
@ RTE_DMA_DIR_DEV_TO_DEV
Definition: rte_dmadev.h:432
int rte_dma_start(int16_t dev_id)
int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info)
int rte_dma_close(int16_t dev_id)
uint16_t rte_dma_count_avail(void)
static int rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length, uint64_t flags)
Definition: rte_dmadev.h:835
int rte_dma_get_dev_id_by_name(const char *name)
int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf)
rte_dma_status_code
Definition: rte_dmadev.h:706
@ RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR
Definition: rte_dmadev.h:751
@ RTE_DMA_STATUS_INVALID_DST_ADDR
Definition: rte_dmadev.h:727
@ RTE_DMA_STATUS_PAGE_FAULT
Definition: rte_dmadev.h:758
@ RTE_DMA_STATUS_INVALID_LENGTH
Definition: rte_dmadev.h:734
@ RTE_DMA_STATUS_NOT_ATTEMPTED
Definition: rte_dmadev.h:723
@ RTE_DMA_STATUS_BUS_READ_ERROR
Definition: rte_dmadev.h:741
@ RTE_DMA_STATUS_INVALID_ADDR
Definition: rte_dmadev.h:732
@ RTE_DMA_STATUS_ERROR_UNKNOWN
Definition: rte_dmadev.h:762
@ RTE_DMA_STATUS_BUS_ERROR
Definition: rte_dmadev.h:747
@ RTE_DMA_STATUS_BUS_WRITE_ERROR
Definition: rte_dmadev.h:743
@ RTE_DMA_STATUS_DATA_POISION
Definition: rte_dmadev.h:749
@ RTE_DMA_STATUS_INVALID_OPCODE
Definition: rte_dmadev.h:739
@ RTE_DMA_STATUS_INVALID_SRC_ADDR
Definition: rte_dmadev.h:725
@ RTE_DMA_STATUS_USER_ABORT
Definition: rte_dmadev.h:715
@ RTE_DMA_STATUS_DEV_LINK_ERROR
Definition: rte_dmadev.h:756
@ RTE_DMA_STATUS_SUCCESSFUL
Definition: rte_dmadev.h:708
static uint16_t rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
Definition: rte_dmadev.h:1087
int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan)
int rte_dma_dev_max(size_t dev_max)
int rte_dma_stop(int16_t dev_id)
static uint16_t rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, uint16_t *last_idx, bool *has_error)
Definition: rte_dmadev.h:994
int rte_dma_dump(int16_t dev_id, FILE *f)
int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan, const struct rte_dma_vchan_conf *conf)
int16_t rte_dma_next_dev(int16_t start_dev_id)
static int rte_dma_submit(int16_t dev_id, uint16_t vchan)
Definition: rte_dmadev.h:957
static uint16_t rte_dma_completed_status(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, uint16_t *last_idx, enum rte_dma_status_code *status)
Definition: rte_dmadev.h:1053
int rte_dma_stats_get(int16_t dev_id, uint16_t vchan, struct rte_dma_stats *stats)
bool rte_dma_is_valid(int16_t dev_id)
rte_dma_port_type
Definition: rte_dmadev.h:440
@ RTE_DMA_PORT_PCIE
Definition: rte_dmadev.h:442
static int rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern, rte_iova_t dst, uint32_t length, uint64_t flags)
Definition: rte_dmadev.h:926
rte_dma_vchan_status
Definition: rte_dmadev.h:664
@ RTE_DMA_VCHAN_HALTED_ERROR
Definition: rte_dmadev.h:667
@ RTE_DMA_VCHAN_ACTIVE
Definition: rte_dmadev.h:666
@ RTE_DMA_VCHAN_IDLE
Definition: rte_dmadev.h:665
static int rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src, struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
Definition: rte_dmadev.h:881
struct rte_mempool * pool
Definition: rte_dmadev.h:538
uint16_t nb_vchans
Definition: rte_dmadev.h:326
bool enable_silent
Definition: rte_dmadev.h:334
uint64_t dev_capa
Definition: rte_dmadev.h:281
uint16_t max_sges
Definition: rte_dmadev.h:295
uint16_t max_vchans
Definition: rte_dmadev.h:283
uint16_t max_desc
Definition: rte_dmadev.h:285
uint16_t min_desc
Definition: rte_dmadev.h:287
const char * dev_name
Definition: rte_dmadev.h:279
uint16_t nb_vchans
Definition: rte_dmadev.h:299
int16_t numa_node
Definition: rte_dmadev.h:297
enum rte_dma_port_type port_type
Definition: rte_dmadev.h:456
__extension__ struct rte_dma_port_param::@112::@114 pcie
uint64_t reserved[2]
Definition: rte_dmadev.h:522
rte_iova_t addr
Definition: rte_dmadev.h:771
uint32_t length
Definition: rte_dmadev.h:772
uint64_t submitted
Definition: rte_dmadev.h:609
uint64_t errors
Definition: rte_dmadev.h:615
uint64_t completed
Definition: rte_dmadev.h:613
enum rte_dma_direction direction
Definition: rte_dmadev.h:555
struct rte_dma_auto_free_param auto_free
Definition: rte_dmadev.h:581
struct rte_dma_port_param src_port
Definition: rte_dmadev.h:565
struct rte_dma_port_param dst_port
Definition: rte_dmadev.h:573