Skip to content

Commit 3dc8570

Browse files
bosilcabosilca
authored
Merge pull request #9848 from hjelmn/update_attributes_for_sessions
ompi/attr: update attribute system to prepare for sessions
2 parents 328f9b3 + 60779e2 commit 3dc8570

File tree

8 files changed

+184
-79
lines changed

8 files changed

+184
-79
lines changed

‎ompi/attribute/attribute.c

Lines changed: 136 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -14,6 +15,8 @@
1415
* reserved.
1516
* Copyright (c) 2017 Research Organization for Information Science
1617
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2018-2021 Triad National Security, LLC. All rights
19+
* reserved.
1720
* $COPYRIGHT$
1821
*
1922
* Additional copyrights may follow
@@ -242,6 +245,7 @@
242245
#include "ompi/win/win.h" /* ompi_win_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
243246
#include "ompi/mpi/fortran/base/fint_2_int.h"
244247

248+
245249
/*
246250
* Macros
247251
*/
@@ -256,9 +260,9 @@
256260
#define attr_datatype_f d_f_to_c_index
257261
#define attr_win_f w_f_to_c_index
258262

259-
#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(key_bitmap, (key))
263+
#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(attr_subsys->key_bitmap, (key))
260264

261-
#define FREE_KEY(key) opal_bitmap_clear_bit(key_bitmap, (key))
265+
#define FREE_KEY(key) opal_bitmap_clear_bit(attr_subsys->key_bitmap, (key))
262266

263267

264268
/* Not checking for NULL_DELETE_FN here, since according to the
@@ -406,13 +410,24 @@ typedef struct attribute_value_t {
406410
int av_sequence;
407411
} attribute_value_t;
408412

413+
/*
414+
* struct to hold state of attr subsys
415+
*/
416+
417+
typedef struct attr_subsys_t {
418+
opal_object_t super;
419+
opal_hash_table_t *keyval_hash;
420+
opal_bitmap_t *key_bitmap;
421+
} attr_subsys_t;
409422

410423
/*
411424
* Local functions
412425
*/
413426
static void attribute_value_construct(attribute_value_t *item);
414427
static void ompi_attribute_keyval_construct(ompi_attribute_keyval_t *keyval);
415428
static void ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval);
429+
static void attr_subsys_construct(attr_subsys_t *subsys);
430+
static void attr_subsys_destruct(attr_subsys_t *subsys);
416431
static int set_value(ompi_attribute_type_t type, void *object,
417432
opal_hash_table_t **attr_hash, int key,
418433
attribute_value_t *new_attr,
@@ -425,6 +440,13 @@ static MPI_Aint translate_to_aint(attribute_value_t *val);
425440

426441
static int compare_attr_sequence(const void *attr1, const void *attr2);
427442

443+
/*
444+
* attribute_subsys_t class
445+
*/
446+
static OBJ_CLASS_INSTANCE(attr_subsys_t,
447+
opal_object_t,
448+
attr_subsys_construct,
449+
attr_subsys_destruct);
428450

429451
/*
430452
* attribute_value_t class
@@ -443,24 +465,31 @@ static OBJ_CLASS_INSTANCE(ompi_attribute_keyval_t,
443465
ompi_attribute_keyval_construct,
444466
ompi_attribute_keyval_destruct);
445467

468+
/*
469+
* compatibility until sessions work is finished
470+
*/
471+
static inline int ompi_mpi_instance_retain(void) {
472+
return OMPI_SUCCESS;
473+
}
474+
475+
static inline void ompi_mpi_instance_release(void) {
476+
}
446477

447478
/*
448479
* Static variables
449480
*/
450481

451-
static opal_hash_table_t *keyval_hash;
452-
static opal_bitmap_t *key_bitmap;
453-
static int attr_sequence;
482+
static attr_subsys_t *attr_subsys = NULL;
454483
static unsigned int int_pos = 12345;
455484
static unsigned int integer_pos = 12345;
485+
static int attr_sequence;
456486

457487
/*
458488
* MPI attributes are *not* high performance, so just use a One Big Lock
459489
* approach. However, this lock is released before a user provided callback is
460490
* triggered and acquired right after, allowing for recursive behaviors.
461491
*/
462-
static opal_mutex_t attribute_lock;
463-
492+
static opal_mutex_t attribute_lock = OPAL_MUTEX_STATIC_INIT;
464493

465494
/*
466495
* attribute_value_t constructor function
@@ -507,33 +536,68 @@ ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval)
507536
free(keyval->bindings_extra_state);
508537
}
509538

510-
opal_hash_table_remove_value_uint32(keyval_hash, keyval->key);
539+
opal_hash_table_remove_value_uint32(attr_subsys->keyval_hash, keyval->key);
511540
FREE_KEY(keyval->key);
512541
}
513542
}
514543

515544

516-
/*
517-
* This will initialize the main list to store key- attribute
518-
* items. This will be called one time, during MPI_INIT().
519-
*/
520-
int ompi_attr_init(void)
545+
int ompi_attr_get_ref(void)
546+
{
547+
int ret = OMPI_SUCCESS;
548+
549+
OPAL_THREAD_LOCK(&attribute_lock);
550+
551+
if (NULL == attr_subsys) {
552+
attr_subsys = OBJ_NEW(attr_subsys_t);
553+
if (NULL == attr_subsys) {
554+
ret = OMPI_ERR_OUT_OF_RESOURCE;
555+
goto fn_exit;
556+
}
557+
if ((NULL == attr_subsys->keyval_hash) || (NULL == attr_subsys->key_bitmap)) {
558+
OBJ_RELEASE(attr_subsys);
559+
attr_subsys = NULL;
560+
ret = OMPI_ERR_OUT_OF_RESOURCE;
561+
goto fn_exit;
562+
}
563+
} else {
564+
OBJ_RETAIN(attr_subsys);
565+
}
566+
567+
fn_exit:
568+
OPAL_THREAD_UNLOCK(&attribute_lock);
569+
570+
return ret;
571+
}
572+
573+
int ompi_attr_put_ref(void)
574+
{
575+
if (NULL != attr_subsys) {
576+
OBJ_RELEASE(attr_subsys);
577+
}
578+
return OMPI_SUCCESS;
579+
}
580+
581+
static void attr_subsys_construct(attr_subsys_t *subsys)
521582
{
522583
int ret;
523584
void *bogus = (void*) 1;
524585
int *p = (int *) &bogus;
525586

526-
keyval_hash = OBJ_NEW(opal_hash_table_t);
527-
if (NULL == keyval_hash) {
528-
return OMPI_ERR_OUT_OF_RESOURCE;
529-
}
530-
key_bitmap = OBJ_NEW(opal_bitmap_t);
587+
subsys->keyval_hash = OBJ_NEW(opal_hash_table_t);
588+
589+
subsys->key_bitmap = OBJ_NEW(opal_bitmap_t);
590+
531591
/*
532592
* Set the max size to OMPI_FORTRAN_HANDLE_MAX to enforce bound
533593
*/
534-
opal_bitmap_set_max_size (key_bitmap, OMPI_FORTRAN_HANDLE_MAX);
535-
if (0 != opal_bitmap_init(key_bitmap, 32)) {
536-
return OMPI_ERR_OUT_OF_RESOURCE;
594+
opal_bitmap_set_max_size (subsys->key_bitmap,
595+
OMPI_FORTRAN_HANDLE_MAX);
596+
ret = opal_bitmap_init(subsys->key_bitmap, 32);
597+
assert(OPAL_SUCCESS == ret);
598+
599+
for (int i = 0; i < MPI_ATTR_PREDEFINED_KEY_MAX; i++) {
600+
opal_bitmap_set_bit(subsys->key_bitmap, i);
537601
}
538602

539603
for (int_pos = 0; int_pos < (sizeof(void*) / sizeof(int));
@@ -550,31 +614,21 @@ int ompi_attr_init(void)
550614
}
551615
}
552616

553-
OBJ_CONSTRUCT(&attribute_lock, opal_mutex_t);
554-
555-
if (OMPI_SUCCESS != (ret = opal_hash_table_init(keyval_hash,
556-
ATTR_TABLE_SIZE))) {
557-
return ret;
558-
}
559-
if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined())) {
560-
return ret;
561-
}
617+
ret = opal_hash_table_init(subsys->keyval_hash, ATTR_TABLE_SIZE);
618+
assert (OPAL_SUCCESS == ret);
562619

563-
return OMPI_SUCCESS;
620+
attr_sequence = 0;
564621
}
565622

566623

567624
/*
568-
* Cleanup everything during MPI_Finalize().
625+
* Cleanup everything when no more refs to the attr subsys
569626
*/
570-
int ompi_attr_finalize(void)
627+
static void attr_subsys_destruct(attr_subsys_t *subsys)
571628
{
572629
ompi_attr_free_predefined();
573-
OBJ_DESTRUCT(&attribute_lock);
574-
OBJ_RELEASE(keyval_hash);
575-
OBJ_RELEASE(key_bitmap);
576-
577-
return OMPI_SUCCESS;
630+
OBJ_RELEASE(subsys->keyval_hash);
631+
OBJ_RELEASE(subsys->key_bitmap);
578632
}
579633

580634
/*****************************************************************************/
@@ -609,10 +663,15 @@ static int ompi_attr_create_keyval_impl(ompi_attribute_type_t type,
609663

610664
/* Create a new unique key and fill the hash */
611665
OPAL_THREAD_LOCK(&attribute_lock);
612-
ret = CREATE_KEY(key);
666+
ret = MPI_SUCCESS;
667+
if (!(flags & OMPI_KEYVAL_PREDEFINED)) {
668+
ret = CREATE_KEY(key);
669+
}
670+
613671
if (OMPI_SUCCESS == ret) {
614672
keyval->key = *key;
615-
ret = opal_hash_table_set_value_uint32(keyval_hash, *key, keyval);
673+
ret = opal_hash_table_set_value_uint32(attr_subsys->keyval_hash,
674+
*key, keyval);
616675
}
617676

618677
if (OMPI_SUCCESS != ret) {
@@ -635,11 +694,22 @@ int ompi_attr_create_keyval(ompi_attribute_type_t type,
635694
void *bindings_extra_state)
636695
{
637696
ompi_attribute_fortran_ptr_t es_tmp;
697+
int rc;
698+
699+
rc = ompi_mpi_instance_retain ();
700+
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
701+
return rc;
702+
}
638703

639704
es_tmp.c_ptr = extra_state;
640-
return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
641-
key, &es_tmp, flags,
642-
bindings_extra_state);
705+
rc = ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
706+
key, &es_tmp, flags,
707+
bindings_extra_state);
708+
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
709+
ompi_mpi_instance_release ();
710+
}
711+
712+
return rc;
643713
}
644714

645715
int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
@@ -651,6 +721,12 @@ int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
651721
void *bindings_extra_state)
652722
{
653723
ompi_attribute_fortran_ptr_t es_tmp;
724+
int rc;
725+
726+
rc = ompi_mpi_instance_retain ();
727+
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
728+
return rc;
729+
}
654730

655731
es_tmp.f_integer = extra_state;
656732
#if SIZEOF_INT == OMPI_SIZEOF_FORTRAN_INTEGER
@@ -670,6 +746,12 @@ int ompi_attr_create_keyval_aint(ompi_attribute_type_t type,
670746
void *bindings_extra_state)
671747
{
672748
ompi_attribute_fortran_ptr_t es_tmp;
749+
int rc;
750+
751+
rc = ompi_mpi_instance_retain ();
752+
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
753+
return rc;
754+
}
673755

674756
es_tmp.f_address = extra_state;
675757
return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
@@ -687,7 +769,7 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
687769

688770
/* Find the key-value pair */
689771
OPAL_THREAD_LOCK(&attribute_lock);
690-
ret = opal_hash_table_get_value_uint32(keyval_hash, *key,
772+
ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, *key,
691773
(void **) &keyval);
692774
if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
693775
(keyval->attr_type != type) ||
@@ -707,6 +789,9 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
707789
opal_atomic_wmb();
708790
OPAL_THREAD_UNLOCK(&attribute_lock);
709791

792+
/* balance out retain in keyval_create */
793+
ompi_mpi_instance_release ();
794+
710795
return MPI_SUCCESS;
711796
}
712797

@@ -720,7 +805,7 @@ int ompi_attr_set_c(ompi_attribute_type_t type, void *object,
720805
opal_hash_table_t **attr_hash,
721806
int key, void *attribute, bool predefined)
722807
{
723-
int ret;
808+
int ret = MPI_SUCCESS;
724809
attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
725810
if (NULL == new_attr) {
726811
return OMPI_ERR_OUT_OF_RESOURCE;
@@ -942,7 +1027,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
9421027

9431028
/* Get the keyval in the main keyval hash - so that we know
9441029
what the copy_attr_fn is */
945-
err = opal_hash_table_get_value_uint32(keyval_hash, key,
1030+
err = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
9461031
(void **) &hash_value);
9471032
if (OMPI_SUCCESS != err) {
9481033
/* This should not happen! */
@@ -1037,7 +1122,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
10371122
attribute_value_t *attr;
10381123

10391124
/* Check if the key is valid in the master keyval hash */
1040-
ret = opal_hash_table_get_value_uint32(keyval_hash, key,
1125+
ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
10411126
(void **) &keyval);
10421127

10431128
if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
@@ -1053,7 +1138,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
10531138
goto exit;
10541139
}
10551140

1056-
/* Check if the key is valid for the communicator/window/dtype. If
1141+
/* Check if the key is valid for the communicator/window/dtype/instance. If
10571142
yes, then delete the attribute and key entry from the object's
10581143
hash */
10591144
ret = opal_hash_table_get_value_uint32(attr_hash, key, (void**) &attr);
@@ -1198,7 +1283,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
11981283
/* Note that this function can be invoked by ompi_attr_copy_all()
11991284
to set attributes on the new object (in addition to the
12001285
top-level MPI_* functions that set attributes). */
1201-
ret = opal_hash_table_get_value_uint32(keyval_hash, key,
1286+
ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
12021287
(void **) &keyval);
12031288

12041289
/* If key not found */
@@ -1242,7 +1327,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
12421327
had_old = true;
12431328
}
12441329

1245-
ret = opal_hash_table_get_value_uint32(keyval_hash, key,
1330+
ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
12461331
(void **) &keyval);
12471332
if ((OMPI_SUCCESS != ret ) || (NULL == keyval)) {
12481333
/* Keyval has disappeared underneath us -- this shouldn't
@@ -1288,7 +1373,7 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
12881373
with the key, then the call is valid and returns FALSE in the
12891374
flag argument */
12901375
*flag = 0;
1291-
ret = opal_hash_table_get_value_uint32(keyval_hash, key,
1376+
ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
12921377
(void**) &keyval);
12931378
if (OMPI_ERR_NOT_FOUND == ret) {
12941379
return MPI_KEYVAL_INVALID;

0 commit comments

Comments
 (0)