Skip to content

Commit af4e7c6

Browse files
wenduwanwenduwan
committed
opal/mca/accelerator: introduce get_device_pci_attr api
Introduce get_device_pci_attr api to query accelerator device PCI attributes. This enables intelligent selection of other PCI(e) devices based on affinity with the accelerator, e.g. NICs. Signed-off-by: Wenduo Wang <[email protected]>
1 parent 5ea8638 commit af4e7c6

File tree

4 files changed

+76
-0
lines changed

4 files changed

+76
-0
lines changed

‎opal/mca/accelerator/accelerator.h

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,15 @@ struct opal_accelerator_stream_t {
110110
void *stream;
111111
};
112112
typedef struct opal_accelerator_stream_t opal_accelerator_stream_t;
113+
114+
struct opal_accelerator_pci_attr_t {
115+
uint16_t domain_id;
116+
uint8_t bus_id;
117+
uint8_t device_id;
118+
uint8_t function_id;
119+
};
120+
typedef struct opal_accelerator_pci_attr_t opal_accelerator_pci_attr_t;
121+
113122
OBJ_CLASS_DECLARATION(opal_accelerator_stream_t);
114123

115124
struct opal_accelerator_event_t {
@@ -346,6 +355,17 @@ typedef int (*opal_accelerator_base_module_host_unregister_fn_t)(
346355
typedef int (*opal_accelerator_base_module_get_device_fn_t)(
347356
int *dev_id);
348357

358+
/**
359+
* Retrieves PCI attributes of an accelerator device.
360+
*
361+
* @param[int] dev_id Accelerator device id
362+
* @param[out] pci_attr PCI attributes of the requested device
363+
*
364+
* @return OPAL_SUCCESS or error status on failure
365+
*/
366+
typedef int (*opal_accelerator_base_module_get_device_pci_attr_fn_t)(
367+
int dev_id, opal_accelerator_pci_attr_t *pci_attr);
368+
349369
/**
350370
* Queries if a device may directly access a peer device's memory.
351371
*
@@ -398,6 +418,7 @@ typedef struct {
398418
opal_accelerator_base_module_host_unregister_fn_t host_unregister;
399419

400420
opal_accelerator_base_module_get_device_fn_t get_device;
421+
opal_accelerator_base_module_get_device_pci_attr_fn_t get_device_pci_attr;
401422
opal_accelerator_base_module_device_can_access_peer_fn_t device_can_access_peer;
402423

403424
opal_accelerator_base_module_get_buffer_id_fn_t get_buffer_id;

‎opal/mca/accelerator/cuda/accelerator_cuda.c

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ static int accelerator_cuda_host_register(int dev_id, void *ptr, size_t size);
4545
static int accelerator_cuda_host_unregister(int dev_id, void *ptr);
4646

4747
static int accelerator_cuda_get_device(int *dev_id);
48+
static int accelerator_cuda_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
4849
static int accelerator_cuda_device_can_access_peer( int *access, int dev1, int dev2);
4950

5051
static int accelerator_cuda_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
@@ -70,6 +71,7 @@ opal_accelerator_base_module_t opal_accelerator_cuda_module =
7071
accelerator_cuda_host_unregister,
7172

7273
accelerator_cuda_get_device,
74+
accelerator_cuda_get_device_pci_attr,
7375
accelerator_cuda_device_can_access_peer,
7476

7577
accelerator_cuda_get_buffer_id
@@ -578,6 +580,45 @@ static int accelerator_cuda_get_device(int *dev_id)
578580
return 0;
579581
}
580582

583+
static int accelerator_cuda_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
584+
{
585+
CUresult result;
586+
int ret;
587+
static const int PCI_BUS_ID_LENGTH = 13;
588+
char pci_bus_id[PCI_BUS_ID_LENGTH];
589+
char domain_id[5] = {0}, bus_id[3] = {0}, device_id[3] = {0}, function_id[2] = {0};
590+
591+
if (NULL == pci_attr) {
592+
return OPAL_ERR_BAD_PARAM;
593+
}
594+
595+
result = cuDeviceGetPCIBusId(pci_bus_id, PCI_BUS_ID_LENGTH, dev_id);
596+
597+
if (CUDA_SUCCESS != result) {
598+
opal_output_verbose(5, opal_accelerator_base_framework.framework_output,
599+
"CUDA: Failed to get device PCI bus id\n");
600+
return OPAL_ERROR;
601+
}
602+
603+
ret = sscanf(pci_bus_id, "%4s:%2s:%2s.%s", domain_id, bus_id, device_id, function_id);
604+
if (4 > ret) {
605+
opal_output_verbose(5, opal_accelerator_base_framework.framework_output,
606+
"CUDA: Failed to parse device PCI bus id\n");
607+
return OPAL_ERROR;
608+
}
609+
610+
errno = 0;
611+
pci_attr->domain_id = strtol(domain_id, NULL, 16);
612+
pci_attr->bus_id = strtol(bus_id, NULL, 16);
613+
pci_attr->device_id = strtol(device_id, NULL, 16);
614+
pci_attr->function_id = strtol(function_id, NULL, 16);
615+
if (0 != errno) {
616+
return OPAL_ERROR;
617+
}
618+
619+
return OPAL_SUCCESS;
620+
}
621+
581622
static int accelerator_cuda_device_can_access_peer(int *access, int dev1, int dev2)
582623
{
583624
CUresult result;

‎opal/mca/accelerator/null/accelerator_null_component.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ static int accelerator_null_host_register(int dev_id, void *ptr, size_t size);
5959
static int accelerator_null_host_unregister(int dev_id, void *ptr);
6060

6161
static int accelerator_null_get_device(int *dev_id);
62+
static int accelerator_null_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
6263
static int accelerator_null_device_can_access_peer(int *access, int dev1, int dev2);
6364

6465
static int accelerator_null_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
@@ -122,6 +123,7 @@ opal_accelerator_base_module_t opal_accelerator_null_module =
122123
accelerator_null_host_unregister,
123124

124125
accelerator_null_get_device,
126+
accelerator_null_get_device_pci_attr,
125127
accelerator_null_device_can_access_peer,
126128

127129
accelerator_null_get_buffer_id
@@ -235,6 +237,11 @@ static int accelerator_null_get_device(int *dev_id)
235237
return OPAL_ERR_NOT_IMPLEMENTED;
236238
}
237239

240+
static int accelerator_null_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
241+
{
242+
return OPAL_ERR_NOT_IMPLEMENTED;
243+
}
244+
238245
static int accelerator_null_device_can_access_peer( int *access, int dev1, int dev2)
239246
{
240247
return OPAL_ERR_NOT_IMPLEMENTED;

‎opal/mca/accelerator/rocm/accelerator_rocm_module.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ static int mca_accelerator_rocm_host_register(int dev_id, void *ptr, size_t size
3737
static int mca_accelerator_rocm_host_unregister(int dev_id, void *ptr);
3838

3939
static int mca_accelerator_rocm_get_device(int *dev_id);
40+
static int mca_accelerator_rocm_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
4041
static int mca_accelerator_rocm_device_can_access_peer( int *access, int dev1, int dev2);
4142

4243
static int mca_accelerator_rocm_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
@@ -62,6 +63,7 @@ opal_accelerator_base_module_t opal_accelerator_rocm_module =
6263
mca_accelerator_rocm_host_unregister,
6364

6465
mca_accelerator_rocm_get_device,
66+
mca_accelerator_rocm_get_device_pci_attr,
6567
mca_accelerator_rocm_device_can_access_peer,
6668

6769
mca_accelerator_rocm_get_buffer_id
@@ -476,6 +478,11 @@ static int mca_accelerator_rocm_get_device(int *dev_id)
476478
return OPAL_SUCCESS;
477479
}
478480

481+
static int mca_accelerator_rocm_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
482+
{
483+
return OPAL_ERR_NOT_IMPLEMENTED;
484+
}
485+
479486
static int mca_accelerator_rocm_device_can_access_peer(int *access, int dev1, int dev2)
480487
{
481488
if (NULL == access || dev1 < 0 || dev2 < 0){

0 commit comments

Comments
 (0)