From 775e6b6156d7399d10f053fa70f3e18a785fdaa9 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 8 Nov 2025 14:33:28 -0600 Subject: [PATCH 01/19] new error core NC_EMULTIDEFINE_HINTS for inconsistent I/O hints set by users --- src/dispatchers/error_codes.c | 4 ++++ src/include/pnetcdf.h.in | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dispatchers/error_codes.c b/src/dispatchers/error_codes.c index e74a20d483..b78c447d4d 100644 --- a/src/dispatchers/error_codes.c +++ b/src/dispatchers/error_codes.c @@ -289,6 +289,8 @@ ncmpi_strerror(int err) return "Variable fill value is inconsistent among processes."; case NC_EMULTIDEFINE_CMODE: return "File create mode is inconsistent among processes."; + case NC_EMULTIDEFINE_HINTS: + return "I/O hints are not consistent among processes."; case NC_EBADLOG: return "Unrecognized burst buffering log file format."; case NC_EFLUSHED: @@ -747,6 +749,8 @@ ncmpi_strerrno(int err) case (NC_EMULTIDEFINE_VAR_FILL_MODE): return "NC_EMULTIDEFINE_VAR_FILL_MODE"; case (NC_EMULTIDEFINE_VAR_FILL_VALUE): return "NC_EMULTIDEFINE_VAR_FILL_VALUE"; case (NC_EMULTIDEFINE_CMODE): return "NC_EMULTIDEFINE_CMODE"; + case (NC_EMULTIDEFINE_HINTS): return "NC_EMULTIDEFINE_HINTS"; + default: sprintf(unknown_str,"Unknown code %d",err); } diff --git a/src/include/pnetcdf.h.in b/src/include/pnetcdf.h.in index df7e9f66a2..6ce7499c61 100644 --- a/src/include/pnetcdf.h.in +++ b/src/include/pnetcdf.h.in @@ -686,9 +686,10 @@ by the desired type. */ #define NC_EMULTIDEFINE_VAR_FILL_MODE (-271) /**< inconsistent variable fill mode */ #define NC_EMULTIDEFINE_VAR_FILL_VALUE (-272) /**< inconsistent variable fill value */ #define NC_EMULTIDEFINE_CMODE (-273) /**< inconsistent file create modes among processes */ +#define NC_EMULTIDEFINE_HINTS (-274) /**< inconsistent I/O hints among processes */ #define NC_EMULTIDEFINE_FIRST NC_EMULTIDEFINE -#define NC_EMULTIDEFINE_LAST NC_EMULTIDEFINE_CMODE +#define NC_EMULTIDEFINE_LAST NC_EMULTIDEFINE_HINTS /* backward compatible with PnetCDF 1.3.1 and earlier */ #define NC_ECMODE NC_EMULTIDEFINE_OMODE From 1709e89814c80a3964479089047f5481d1975131 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 8 Nov 2025 14:37:32 -0600 Subject: [PATCH 02/19] rename hint variables used internally cb_read -> romio_cb_read cb_write -> romio_cb_write ds_read -> romio_ds_read ds_write -> romio_ds_write no_indep_rw -> romio_no_indep_rw --- src/drivers/pncio/pncio.h | 25 +++++++++++++------------ src/drivers/pncio/pncio_lustre_open.c | 20 ++++++++++---------- src/drivers/pncio/pncio_lustre_wrcoll.c | 22 +++++++++++----------- src/drivers/pncio/pncio_lustre_wrstr.c | 24 ++++++++++++------------ src/drivers/pncio/pncio_read_coll.c | 8 ++++---- src/drivers/pncio/pncio_read_str.c | 2 +- src/drivers/pncio/pncio_write_coll.c | 10 +++++----- src/drivers/pncio/pncio_write_str.c | 20 ++++++++++---------- 8 files changed, 66 insertions(+), 65 deletions(-) diff --git a/src/drivers/pncio/pncio.h b/src/drivers/pncio/pncio.h index 035febd796..f7c3e10a04 100644 --- a/src/drivers/pncio/pncio.h +++ b/src/drivers/pncio/pncio.h @@ -110,24 +110,25 @@ enum { typedef struct { int striping_factor; int striping_unit; - int cb_read; - int cb_write; + int start_iodevice; int cb_nodes; int cb_buffer_size; - int ds_read; - int ds_write; - int no_indep_rw; int ind_rd_buffer_size; int ind_wr_buffer_size; - int start_iodevice; + + int romio_cb_read; + int romio_cb_write; + int romio_ds_read; + int romio_ds_write; + int romio_no_indep_rw; + + /* Hints for Lustre file system */ + int lustre_overstriping_ratio; + + /* Hints set by PnetCDF internally */ + int lustre_num_osts; int *ranklist; - union { - struct { - int num_osts; - int overstriping_ratio; - } lustre; - } fs_hints; } PNCIO_Hints; typedef struct { diff --git a/src/drivers/pncio/pncio_lustre_open.c b/src/drivers/pncio/pncio_lustre_open.c index d9a1692a6c..687b3e5910 100644 --- a/src/drivers/pncio/pncio_lustre_open.c +++ b/src/drivers/pncio/pncio_lustre_open.c @@ -826,7 +826,7 @@ assert(mpi_io_mode & MPI_MODE_CREATE); str_unit = fd->hints->striping_unit; str_factor = fd->hints->striping_factor; start_iodev = fd->hints->start_iodevice; - overstriping_ratio = fd->hints->fs_hints.lustre.overstriping_ratio; + overstriping_ratio = fd->hints->lustre_overstriping_ratio; /* obtain the total number of OSTs available */ total_num_OSTs = get_total_avail_osts(fd->filename); @@ -847,7 +847,7 @@ assert(mpi_io_mode & MPI_MODE_CREATE); * fd->hints->striping_factor = 0; * fd->hints->striping_unit = 0; * fd->hints->start_iodevice = -1; - * fd->hints->fs_hints.lustre.overstriping_ratio = 1; + * fd->hints->lustre_overstriping_ratio = 1; */ /* In many cases, the Lustre striping configuration of the file to be @@ -1017,8 +1017,8 @@ assert(mpi_io_mode & MPI_MODE_CREATE); fd->hints->striping_factor = stripin_info[1]; fd->hints->start_iodevice = stripin_info[2]; if (fd->file_system == PNCIO_LUSTRE) { - fd->hints->fs_hints.lustre.num_osts = stripin_info[3]; - fd->hints->fs_hints.lustre.overstriping_ratio = stripin_info[1] / stripin_info[3]; + fd->hints->lustre_num_osts = stripin_info[3]; + fd->hints->lustre_overstriping_ratio = stripin_info[1] / stripin_info[3]; } if (rank > 0) { /* non-root processes */ @@ -1035,10 +1035,10 @@ assert(mpi_io_mode & MPI_MODE_CREATE); MPI_Info_set(fd->info, "romio_filesystem_type", "LUSTRE:"); - snprintf(int_str, 16, "%d", fd->hints->fs_hints.lustre.num_osts); + snprintf(int_str, 16, "%d", fd->hints->lustre_num_osts); MPI_Info_set(fd->info, "lustre_num_osts", int_str); - snprintf(int_str, 16, "%d", fd->hints->fs_hints.lustre.overstriping_ratio); + snprintf(int_str, 16, "%d", fd->hints->lustre_overstriping_ratio); MPI_Info_set(fd->info, "lustre_overstriping_ratio", int_str); return err; @@ -1119,18 +1119,18 @@ static int wkl=0; if (wkl == 0 && rank == 0) { printf("\nxxxx %s at %d: %s ---- fd->hints->striping_unit = stripin_info[0]; fd->hints->striping_factor = stripin_info[1]; fd->hints->start_iodevice = stripin_info[2]; - fd->hints->fs_hints.lustre.num_osts = stripin_info[3]; - fd->hints->fs_hints.lustre.overstriping_ratio = stripin_info[1] / stripin_info[3]; + fd->hints->lustre_num_osts = stripin_info[3]; + fd->hints->lustre_overstriping_ratio = stripin_info[1] / stripin_info[3]; /* construct cb_nodes rank list */ Lustre_set_cb_node_list(fd); MPI_Info_set(fd->info, "romio_filesystem_type", "LUSTRE:"); - snprintf(int_str, 16, "%d", fd->hints->fs_hints.lustre.num_osts); + snprintf(int_str, 16, "%d", fd->hints->lustre_num_osts); MPI_Info_set(fd->info, "lustre_num_osts", int_str); - snprintf(int_str, 16, "%d", fd->hints->fs_hints.lustre.overstriping_ratio); + snprintf(int_str, 16, "%d", fd->hints->lustre_overstriping_ratio); MPI_Info_set(fd->info, "lustre_overstriping_ratio", int_str); return err; diff --git a/src/drivers/pncio/pncio_lustre_wrcoll.c b/src/drivers/pncio/pncio_lustre_wrcoll.c index 03b0a59e94..40192f1862 100644 --- a/src/drivers/pncio/pncio_lustre_wrcoll.c +++ b/src/drivers/pncio/pncio_lustre_wrcoll.c @@ -638,7 +638,7 @@ double curT = MPI_Wtime(); if (buf_view.count > 1) buf_view.rem = buf_view.len[0]; - if (fd->hints->cb_write == PNCIO_HINT_DISABLE) { + if (fd->hints->romio_cb_write == PNCIO_HINT_DISABLE) { /* collective write is explicitly disabled by user */ do_collect = 0; } @@ -700,14 +700,14 @@ double curT = MPI_Wtime(); NCI_Free(st_end_all); // if (myrank==0) printf("%s %d: do_collect=%d is_interleaved=%d buf_view size=%lld count=%lld is_contig=%d start_offset=%lld end_offset=%lld\n",__func__,__LINE__, do_collect,is_interleaved,buf_view.size,buf_view.count,buf_view.is_contig, start_offset,end_offset); - if (fd->hints->cb_write == PNCIO_HINT_ENABLE) { + if (fd->hints->romio_cb_write == PNCIO_HINT_ENABLE) { /* explicitly enabled by user */ do_collect = 1; } - else if (fd->hints->cb_write == PNCIO_HINT_AUTO) { + else if (fd->hints->romio_cb_write == PNCIO_HINT_AUTO) { // if (myrank==0) printf("%s %d: large_indv_req=%d cb_nodes=%d striping_factor=%d\n",__func__,__LINE__, large_indv_req,fd->hints->cb_nodes , fd->hints->striping_factor); /* Check if collective write is actually necessary, only when - * cb_write hint is set to PNCIO_HINT_AUTO. + * romio_cb_write hint is set to PNCIO_HINT_AUTO. * * Two typical access patterns can benefit from collective write. * 1) access file regions of all processes are interleaved, and @@ -811,7 +811,7 @@ double curT = MPI_Wtime(); */ LUSTRE_Calc_my_req(fd, buf_view.is_contig, &my_req, buf_idx); - if (fd->hints->ds_write != PNCIO_HINT_DISABLE) { + if (fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) { /* When data sieving is considered, below check the current file size * first. If the aggregate access region of this collective write is * beyond the current file size, then we can safely skip the read of @@ -1952,12 +1952,12 @@ int Exchange_data_recv( #endif srt_off_len->off[0] = others_req[j].offsets[start_pos[j]]; srt_off_len->len[0] = others_req[j].lens[start_pos[j]]; - } else if (fd->hints->ds_write == PNCIO_HINT_ENABLE) { + } else if (fd->hints->romio_ds_write == PNCIO_HINT_ENABLE) { /* skip building of srt_off_len and proceed to read-modify-write */ build_srt_off_len = 0; /* assuming there are holes */ hole = 1; - } else if (fd->hints->ds_write == PNCIO_HINT_AUTO) { + } else if (fd->hints->romio_ds_write == PNCIO_HINT_AUTO) { if (DO_HEAP_MERGE(nprocs_recv, srt_off_len->num)) { /* When the number of sorted offset-length lists or the total * number of offset-length pairs are too large, the heap-merge sort @@ -1983,7 +1983,7 @@ int Exchange_data_recv( fd->write_counter[6] = MAX(fd->write_counter[6], nprocs_recv); } #endif - } else { /* if (fd->hints->ds_write == PNCIO_HINT_DISABLE) */ + } else { /* if (fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) */ /* User explicitly disable data sieving to skip read-modify-write. * Whether or not there is a hole is not important. However, * srt_off_len must be constructed to merge all others_req[] into a @@ -2027,11 +2027,11 @@ int Exchange_data_recv( hole = (srt_off_len->num > 1); } -// printf("%s at %d: ds_write=%s build_srt_off_len=%d hole=%d skip_read=%d srt_off_len->num=%lld\n",__func__,__LINE__, (fd->hints->ds_write == PNCIO_HINT_ENABLE)?"ENABLE": (fd->hints->ds_write == PNCIO_HINT_DISABLE)?"DISABLE":"AUTO", build_srt_off_len,hole,fd->skip_read,srt_off_len->num); -// printf("%s at %d: ds_write=%s build_srt_off_len=%d hole=%d nprocs_recv=%d(PNCIO_DS_WR_NAGGRS_LB=%d) numx=%lld(PNCIO_DS_WR_NPAIRS_LB=%d)\n",__func__,__LINE__, (fd->hints->ds_write == PNCIO_HINT_ENABLE)?"ENABLE": (fd->hints->ds_write == PNCIO_HINT_DISABLE)?"DISABLE":"AUTO", build_srt_off_len,hole,nprocs_recv,PNCIO_DS_WR_NAGGRS_LB,numx,PNCIO_DS_WR_NPAIRS_LB); +// printf("%s at %d: romio_ds_write=%s build_srt_off_len=%d hole=%d skip_read=%d srt_off_len->num=%lld\n",__func__,__LINE__, (fd->hints->romio_ds_write == PNCIO_HINT_ENABLE)?"ENABLE": (fd->hints->romio_ds_write == PNCIO_HINT_DISABLE)?"DISABLE":"AUTO", build_srt_off_len,hole,fd->skip_read,srt_off_len->num); +// printf("%s at %d: romio_ds_write=%s build_srt_off_len=%d hole=%d nprocs_recv=%d(PNCIO_DS_WR_NAGGRS_LB=%d) numx=%lld(PNCIO_DS_WR_NPAIRS_LB=%d)\n",__func__,__LINE__, (fd->hints->romio_ds_write == PNCIO_HINT_ENABLE)?"ENABLE": (fd->hints->romio_ds_write == PNCIO_HINT_DISABLE)?"DISABLE":"AUTO", build_srt_off_len,hole,nprocs_recv,PNCIO_DS_WR_NAGGRS_LB,numx,PNCIO_DS_WR_NPAIRS_LB); /* data sieving */ - if (fd->hints->ds_write != PNCIO_HINT_DISABLE && hole) { + if (fd->hints->romio_ds_write != PNCIO_HINT_DISABLE && hole) { if (fd->skip_read) memset(write_buf, 0, range_size); else { diff --git a/src/drivers/pncio/pncio_lustre_wrstr.c b/src/drivers/pncio/pncio_lustre_wrstr.c index 341fab4009..fbd3c75d12 100644 --- a/src/drivers/pncio/pncio_lustre_wrstr.c +++ b/src/drivers/pncio/pncio_lustre_wrstr.c @@ -15,7 +15,7 @@ if (writebuf_len) { \ w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, \ writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ if (w_len < 0) { \ NCI_Free(writebuf); \ @@ -29,7 +29,7 @@ writebuf_len = MIN(end_offset - writebuf_off + 1, \ (writebuf_off / stripe_size + 1) * stripe_size \ - writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ r_len = PNCIO_ReadContig(fd, writebuf, writebuf_len, writebuf_off); \ if (r_len < 0) { \ @@ -42,7 +42,7 @@ write_sz); \ while (write_sz != req_len) { \ w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ if (w_len < 0) { \ NCI_Free(writebuf); \ @@ -56,7 +56,7 @@ writebuf_len = MIN(end_offset - writebuf_off + 1, \ (writebuf_off / stripe_size + 1) * stripe_size \ - writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ r_len = PNCIO_ReadContig(fd, writebuf, writebuf_len, writebuf_off); \ if (r_len < 0) { \ @@ -127,7 +127,7 @@ MPI_Offset PNCIO_LUSTRE_WriteStrided(PNCIO_File *fd, // printf("%s at %d:\n",__func__,__LINE__); - if (fd->hints->ds_write == PNCIO_HINT_DISABLE) { + if (fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) { /* if user has disabled data sieving on writes, use naive * approach instead. */ @@ -162,7 +162,7 @@ if (fd->flat_file.count > 0) assert(offset == 0); /* not whole file visible */ /* if atomicity is true or data sieving is not disable, lock the region * to be accessed */ - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize); for (i = 0; i < buf_view.count; i++) { @@ -176,7 +176,7 @@ if (fd->flat_file.count > 0) assert(offset == 0); /* not whole file visible */ /* write the buffer out the last round */ w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, start_off, SEEK_SET, bufsize); NCI_Free(writebuf); @@ -223,13 +223,13 @@ assert(disp == 0); BUFFERED_WRITE_WITHOUT_READ; /* write the buffer out the last round */ - if (fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); if (w_len > 0) total_w_len += w_len; - if (fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); NCI_Free(writebuf); @@ -255,7 +255,7 @@ assert(j < fd->flat_file.count); /* if atomicity is true or data sieving is not disable, lock the region * to be accessed */ - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); writebuf_off = 0; @@ -348,12 +348,12 @@ assert(k < buf_view.count); /* write the buffer out the last round */ if (writebuf_len) { w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); if (w_len < 0) return w_len; total_w_len += w_len; } - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, start_off, SEEK_SET, end_offset - start_off + 1); NCI_Free(writebuf); diff --git a/src/drivers/pncio/pncio_read_coll.c b/src/drivers/pncio/pncio_read_coll.c index 78af29b482..10ab05933b 100644 --- a/src/drivers/pncio/pncio_read_coll.c +++ b/src/drivers/pncio/pncio_read_coll.c @@ -83,8 +83,8 @@ double curT = MPI_Wtime(); /* number of aggregators, cb_nodes, is stored in the hints */ nprocs_for_coll = fd->hints->cb_nodes; - /* only check for interleaving if cb_read isn't disabled */ - if (fd->hints->cb_read != PNCIO_HINT_DISABLE) { + /* only check for interleaving if romio_cb_read isn't disabled */ + if (fd->hints->romio_cb_read != PNCIO_HINT_DISABLE) { /* For this process's request, calculate the file start and end * offsets. Note: end_offset points to the last byte-offset that will * be accessed, e.g., if start_offset=0 and 100 bytes to be read, @@ -124,8 +124,8 @@ double curT = MPI_Wtime(); interleave_count++; } - if (fd->hints->cb_read == PNCIO_HINT_DISABLE - || (!interleave_count && (fd->hints->cb_read == PNCIO_HINT_AUTO))) { + if (fd->hints->romio_cb_read == PNCIO_HINT_DISABLE + || (!interleave_count && (fd->hints->romio_cb_read == PNCIO_HINT_AUTO))) { /* switch to independent read */ if (st_offsets != NULL) NCI_Free(st_offsets); diff --git a/src/drivers/pncio/pncio_read_str.c b/src/drivers/pncio/pncio_read_str.c index ae554c2fe0..efbfe1a49f 100644 --- a/src/drivers/pncio/pncio_read_str.c +++ b/src/drivers/pncio/pncio_read_str.c @@ -55,7 +55,7 @@ MPI_Offset PNCIO_GEN_ReadStrided(PNCIO_File *fd, // printf("%s at %d:\n",__func__,__LINE__); - if (fd->hints->ds_read == PNCIO_HINT_DISABLE) { + if (fd->hints->romio_ds_read == PNCIO_HINT_DISABLE) { /* if user has disabled data sieving on reads, use naive * approach instead. */ diff --git a/src/drivers/pncio/pncio_write_coll.c b/src/drivers/pncio/pncio_write_coll.c index d6126c5cea..f00ef3bf9d 100644 --- a/src/drivers/pncio/pncio_write_coll.c +++ b/src/drivers/pncio/pncio_write_coll.c @@ -83,8 +83,8 @@ double curT = MPI_Wtime(); */ nprocs_for_coll = fd->hints->cb_nodes; - /* only check for interleaving if cb_write isn't disabled */ - if (fd->hints->cb_write != PNCIO_HINT_DISABLE) { + /* only check for interleaving if romio_cb_write isn't disabled */ + if (fd->hints->romio_cb_write != PNCIO_HINT_DISABLE) { /* For this process's request, calculate the file start and end * offsets. Note: end_offset points to the last byte-offset that will * be accessed, e.g., if start_offset=0 and 100 bytes to be read, @@ -127,11 +127,11 @@ double curT = MPI_Wtime(); interleave_count++; } - if (fd->hints->cb_write == PNCIO_HINT_DISABLE || - (!interleave_count && (fd->hints->cb_write == PNCIO_HINT_AUTO))) { + if (fd->hints->romio_cb_write == PNCIO_HINT_DISABLE || + (!interleave_count && (fd->hints->romio_cb_write == PNCIO_HINT_AUTO))) { /* use independent accesses */ - if (fd->hints->cb_write != PNCIO_HINT_DISABLE) + if (fd->hints->romio_cb_write != PNCIO_HINT_DISABLE) NCI_Free(st_offsets); if (buf_view.size == 0) return 0; diff --git a/src/drivers/pncio/pncio_write_str.c b/src/drivers/pncio/pncio_write_str.c index cb4ac25e84..6cc1f555f0 100644 --- a/src/drivers/pncio/pncio_write_str.c +++ b/src/drivers/pncio/pncio_write_str.c @@ -14,14 +14,14 @@ if (writebuf_len) { \ w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, \ writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ if (w_len < 0) goto fn_exit; \ total_w_len += w_len; \ } \ writebuf_off = req_off; \ writebuf_len = MIN(max_bufsize,end_offset-writebuf_off+1); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ r_len = PNCIO_ReadContig(fd, writebuf, writebuf_len, writebuf_off); \ if (r_len < 0) goto fn_exit; \ @@ -30,7 +30,7 @@ memcpy(writebuf+req_off-writebuf_off, (char*)buf +userbuf_off, write_sz); \ while (write_sz != req_len) { \ w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ if (w_len < 0) goto fn_exit; \ total_w_len += w_len; \ @@ -38,7 +38,7 @@ userbuf_off += write_sz; \ writebuf_off += writebuf_len; \ writebuf_len = MIN(max_bufsize,end_offset-writebuf_off+1); \ - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) \ + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) \ PNCIO_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ r_len = PNCIO_ReadContig(fd, writebuf, writebuf_len, writebuf_off); \ if (r_len < 0) goto fn_exit; \ @@ -70,7 +70,7 @@ MPI_Offset PNCIO_GEN_WriteStrided(PNCIO_File *fd, */ assert(!(buf_view.is_contig && fd->flat_file.is_contig)); - if (fd->hints->ds_write == PNCIO_HINT_DISABLE) { + if (fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) { /* If user has disabled data sieving on reads, use naive approach * instead. */ @@ -104,7 +104,7 @@ assert(fd->disp == 0); /* if atomicity is true or data sieving is not disable, lock the region * to be accessed */ - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); for (i = 0; i < buf_view.count; i++) { @@ -150,7 +150,7 @@ assert(fd->disp == 0); else w_len = 0; - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, start_off, SEEK_SET, end_offset - start_off + 1); if (w_len < 0) @@ -218,7 +218,7 @@ assert(offset == abs_off_in_filetype); /* if atomicity is true or data sieving is not disable, lock the region * to be accessed */ - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1); writebuf_off = 0; @@ -311,12 +311,12 @@ assert(k < buf_view.count); /* write the buffer out finally */ if (writebuf_len) { w_len = PNCIO_WriteContig(fd, writebuf, writebuf_len, writebuf_off); - if (!fd->atomicity && fd->hints->ds_write == PNCIO_HINT_DISABLE) + if (!fd->atomicity && fd->hints->romio_ds_write == PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); if (w_len < 0) goto fn_exit; total_w_len += w_len; } - if (fd->atomicity || fd->hints->ds_write != PNCIO_HINT_DISABLE) + if (fd->atomicity || fd->hints->romio_ds_write != PNCIO_HINT_DISABLE) PNCIO_UNLOCK(fd, start_off, SEEK_SET, end_offset - start_off + 1); } From af3f09590a22017eacb2d96be23a21b066f21d15 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 8 Nov 2025 14:40:13 -0600 Subject: [PATCH 03/19] One Bcast and Allreduce to check consistency of all I/O hints --- src/drivers/pncio/pncio.h | 8 +- src/drivers/pncio/pncio_hints.c | 324 +++++++++++++------------------- 2 files changed, 135 insertions(+), 197 deletions(-) diff --git a/src/drivers/pncio/pncio.h b/src/drivers/pncio/pncio.h index f7c3e10a04..9c4e4530bd 100644 --- a/src/drivers/pncio/pncio.h +++ b/src/drivers/pncio/pncio.h @@ -101,11 +101,9 @@ #define PNCIO_TYPE_OVERLAP 0x00000002 /* if contains overlapping regions */ #define PNCIO_TYPE_NEGATIVE 0x00000004 /* if one of displacements is negative */ -enum { - PNCIO_HINT_AUTO = 0, - PNCIO_HINT_ENABLE = 1, - PNCIO_HINT_DISABLE = 2 -}; +#define PNCIO_HINT_AUTO -1 +#define PNCIO_HINT_DISABLE 0 +#define PNCIO_HINT_ENABLE 1 typedef struct { int striping_factor; diff --git a/src/drivers/pncio/pncio_hints.c b/src/drivers/pncio/pncio_hints.c index 0e7db29a3c..4b20016da4 100644 --- a/src/drivers/pncio/pncio_hints.c +++ b/src/drivers/pncio/pncio_hints.c @@ -19,175 +19,113 @@ #include #include "pncio.h" -/*----< PNCIO_File_get_info() >-----------------------------------------------*/ -int PNCIO_File_get_info(PNCIO_File *fd, - MPI_Info *info_used) -{ - int err; - - err = MPI_Info_dup(fd->info, info_used); - if (err == MPI_SUCCESS) - err = NC_NOERR; - else - err = ncmpii_error_mpi2nc(err, "MPI_Info_dup"); - - return err; +#define GET_INFO_INT(key) { \ + MPI_Info_get(users_info, #key, MPI_MAX_INFO_VAL, value, &flag); \ + if (flag) { \ + MPI_Info_set(fd->info, #key, value); \ + fd->hints->key = atoi(value); \ + } \ } -/*----< Info_check_and_install_int() >---------------------------------------*/ -static -int Info_check_and_install_int(PNCIO_File *fd, - MPI_Info info, - const char *key, - int *local_cache) -{ - int intval, tmp_val, flag, ret = 0; - char value[MPI_MAX_INFO_VAL + 1]; +#define GET_INFO_STR(key) { \ + MPI_Info_get(users_info, #key, MPI_MAX_INFO_VAL, value, &flag); \ + if (flag) { \ + MPI_Info_set(fd->info, #key, value); \ + if (!strcasecmp(value, "true")) \ + fd->hints->key = PNCIO_HINT_ENABLE; \ + else if (!strcasecmp(value, "false")) \ + fd->hints->key = PNCIO_HINT_DISABLE; \ + else if (!strcasecmp(value, "automatic")) \ + fd->hints->key = PNCIO_HINT_AUTO; \ + else if (!strcasecmp(value, "enable")) \ + fd->hints->key = PNCIO_HINT_ENABLE; \ + else if (!strcasecmp(value, "disable")) \ + fd->hints->key = PNCIO_HINT_DISABLE; \ + } \ +} - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - if (flag) { - intval = atoi(value); - tmp_val = intval; - - MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); - /* --BEGIN ERROR HANDLING-- */ - if (tmp_val != intval) { - ret = ncmpii_error_mpi2nc(MPI_ERR_NOT_SAME, __func__); - goto fn_exit; - } - /* --END ERROR HANDLING-- */ - - MPI_Info_set(fd->info, key, value); - /* some file systems do not cache hints in the fd struct */ - if (local_cache != NULL) - *local_cache = intval; - } -fn_exit: - return ret; +#ifdef PNETCDF_DEBUG +#define CHECK_HINT(hint) { \ + if (fd->hints->hint != root_hints->hint) { \ + char int_str[16]; \ + fprintf(stderr, "Error: inconsistent I/O hint %s (%d at rank %d, %d at root)\n", \ + #hint, fd->hints->hint, root_hints->hint); \ + /* overwrite local's hint with root's */ \ + snprintf(int_str, 16, "%d", root_hints->hint); \ + MPI_Info_set(fd->info, #hint, int_str); \ + err = NC_EMULTIDEFINE_HINTS; \ + } \ } +#else +#define CHECK_HINT(hint) { \ + if (fd->hints->hint != root_hints->hint) { \ + /* overwrite local's hint with root's */ \ + char int_str[16]; \ + snprintf(int_str, 16, "%d", root_hints->hint); \ + MPI_Info_set(fd->info, #hint, int_str); \ + err = NC_EMULTIDEFINE_HINTS; \ + } \ +} +#endif -/*----< Info_check_and_install_enabled() >-----------------------------------*/ +/*----< hint_consistency_check() >-------------------------------------------*/ static -int Info_check_and_install_enabled(PNCIO_File *fd, - MPI_Info info, - const char *key, - int *local_cache) +int hint_consistency_check(PNCIO_File *fd) { - int tmp_val, flag, ret = 0; - char value[MPI_MAX_INFO_VAL + 1]; + int err, rank; - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - if (flag) { - if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) { - MPI_Info_set(fd->info, key, value); - *local_cache = PNCIO_HINT_ENABLE; - } else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) { - MPI_Info_set(fd->info, key, value); - *local_cache = PNCIO_HINT_DISABLE; - } else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC")) { - MPI_Info_set(fd->info, key, value); - *local_cache = PNCIO_HINT_AUTO; - /* treat the user-provided string like "enabled": either it is a - * hint ROMIO knows about and can support it, or ROMIO will not - * return the hint at all in the MPI_File_get_info info object - */ - } else if (!strcmp(value, "requested") || !strcmp(value, "REQUESTED")) { - MPI_Info_set(fd->info, key, "enable"); - *local_cache = PNCIO_HINT_ENABLE; - } - - tmp_val = *local_cache; - - MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); - /* --BEGIN ERROR HANDLING-- */ - if (tmp_val != *local_cache) { - ret = ncmpii_error_mpi2nc(MPI_ERR_NOT_SAME, __func__); - goto fn_exit; - } - /* --END ERROR HANDLING-- */ - } -fn_exit: - return ret; -} + MPI_Comm_rank(fd->comm, &rank); -/*----< Info_check_and_install_true() >--------------------------------------*/ -static -int Info_check_and_install_true(PNCIO_File *fd, - MPI_Info info, - const char *key, - int *local_cache) -{ - int flag, tmp_val, ret = 0; - char value[MPI_MAX_INFO_VAL + 1]; + err = NC_NOERR; - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - if (flag) { - if (!strcmp(value, "true") || !strcmp(value, "TRUE")) { - MPI_Info_set(fd->info, key, value); - *local_cache = 1; - } else if (!strcmp(value, "false") || !strcmp(value, "FALSE")) { - MPI_Info_set(fd->info, key, value); - *local_cache = 0; - } - tmp_val = *local_cache; - - MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); - /* --BEGIN ERROR HANDLING-- */ - if (tmp_val != *local_cache) { - ret = ncmpii_error_mpi2nc(MPI_ERR_NOT_SAME, __func__); - goto fn_exit; - } - /* --END ERROR HANDLING-- */ + if (rank == 0) + /* broadcast root's hints */ + MPI_Bcast(fd->hints, sizeof(PNCIO_Hints), MPI_BYTE, 0, fd->comm); + else { + PNCIO_Hints *root_hints; + root_hints = (PNCIO_Hints*) NCI_Malloc(sizeof(PNCIO_Hints)); + + /* broadcast root's hints */ + MPI_Bcast(root_hints, sizeof(PNCIO_Hints), MPI_BYTE, 0, fd->comm); + + /* check hints individually against root's */ + CHECK_HINT(striping_factor); + CHECK_HINT(striping_unit); + CHECK_HINT(start_iodevice); + CHECK_HINT(cb_nodes); + CHECK_HINT(cb_buffer_size); + CHECK_HINT(ind_rd_buffer_size); + CHECK_HINT(ind_wr_buffer_size); + + CHECK_HINT(romio_cb_read); + CHECK_HINT(romio_cb_write); + CHECK_HINT(romio_ds_read); + CHECK_HINT(romio_ds_write); + CHECK_HINT(romio_no_indep_rw); + + CHECK_HINT(lustre_overstriping_ratio); + + NCI_Free(root_hints); } -fn_exit: - return ret; -} -#if 0 -/*----< Info_check_and_install_str() >---------------------------------------*/ -static -int Info_check_and_install_str(PNCIO_File *fd, - MPI_Info info, - const char *key, - char **local_cache) -{ - int flag, ret = 0; - size_t len; - char value[MPI_MAX_INFO_VAL + 1]; + /* All NetCDF erro codes are negative */ + MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MIN, fd->comm); - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - if (flag) { - MPI_Info_set(fd->info, key, value); - len = (strlen(value) + 1) * sizeof(char); - *local_cache = NCI_Malloc(len); - if (*local_cache == NULL) { - ret = NC_ENOMEM; - goto fn_exit; - } - strncpy(*local_cache, value, len); - } -fn_exit: - return ret; + return err; } -#endif /*----< PNCIO_File_SetInfo() >------------------------------------------------*/ /* For PnetCDF, a file info object can only be passed to PnetCDF at file create * or open call, i.e. I/O hints cannot be changed after file create/open. * - * When users_info == MPI_INFO_NULL, this subroutine is an independent call. - * When users_info != MPI_INFO_NULL, this subroutine is a collective call, - * because it calls Info_check_and_install_xxx(), which checks the consistency - * of all hints values set in user's info object. - * - * TODO: instead of sync each hint, a better implementation is to have root - * bcast all hints and let each process checks inconsistency locally. + * This subroutine is a collective call, because it checks consistency of all + * hints among all processes. */ int PNCIO_File_SetInfo(PNCIO_File *fd, MPI_Info users_info) { - int nprocs; + int err=NC_NOERR, flag, nprocs; char value[MPI_MAX_INFO_VAL + 1]; if (users_info == MPI_INFO_NULL) @@ -206,16 +144,16 @@ PNCIO_File_SetInfo(PNCIO_File *fd, * collective buffering */ MPI_Info_set(fd->info, "romio_cb_read", "automatic"); - fd->hints->cb_read = PNCIO_HINT_AUTO; + fd->hints->romio_cb_read = PNCIO_HINT_AUTO; MPI_Info_set(fd->info, "romio_cb_write", "automatic"); - fd->hints->cb_write = PNCIO_HINT_AUTO; + fd->hints->romio_cb_write = PNCIO_HINT_AUTO; /* cb_nodes may be set later right after file open call */ fd->hints->cb_nodes = 0; /* hint indicating that no indep. I/O will be performed on this file */ MPI_Info_set(fd->info, "romio_no_indep_rw", "false"); - fd->hints->no_indep_rw = 0; + fd->hints->romio_no_indep_rw = 0; /* buffer size for data sieving in independent reads */ MPI_Info_set(fd->info, "ind_rd_buffer_size", PNCIO_IND_RD_BUFFER_SIZE_DFLT); @@ -229,9 +167,9 @@ PNCIO_File_SetInfo(PNCIO_File *fd, * sieving */ MPI_Info_set(fd->info, "romio_ds_read", "automatic"); - fd->hints->ds_read = PNCIO_HINT_AUTO; + fd->hints->romio_ds_read = PNCIO_HINT_AUTO; MPI_Info_set(fd->info, "romio_ds_write", "automatic"); - fd->hints->ds_write = PNCIO_HINT_AUTO; + fd->hints->romio_ds_write = PNCIO_HINT_AUTO; /* File striping parameters will be retrieved from the file system set, * once the file is opened. These parameters can also be customized by @@ -242,51 +180,43 @@ PNCIO_File_SetInfo(PNCIO_File *fd, fd->hints->striping_factor = 0; fd->hints->start_iodevice = -1; /* Lustre overstriping ratio. 0 or 1 means disabled */ - fd->hints->fs_hints.lustre.overstriping_ratio = 1; + fd->hints->lustre_overstriping_ratio = 1; /* add in user's info --------------------------------------------------*/ - Info_check_and_install_int(fd, users_info, "cb_buffer_size", - &fd->hints->cb_buffer_size); + + /* size of internal buffer to be used in collective reads and writes */ + GET_INFO_INT(cb_buffer_size); /* enable/disable collective buffering */ - Info_check_and_install_enabled(fd, users_info, "romio_cb_read", - &fd->hints->cb_read); - if (fd->hints->cb_read == PNCIO_HINT_DISABLE) { - /* romio_cb_read overrides no_indep_rw */ + GET_INFO_STR(romio_cb_read); + if (fd->hints->romio_cb_read == PNCIO_HINT_DISABLE) { + /* romio_cb_read overrides romio_no_indep_rw */ MPI_Info_set(fd->info, "romio_no_indep_rw", "false"); - fd->hints->no_indep_rw = PNCIO_HINT_DISABLE; + fd->hints->romio_no_indep_rw = PNCIO_HINT_DISABLE; } - Info_check_and_install_enabled(fd, users_info, "romio_cb_write", - &fd->hints->cb_write); - if (fd->hints->cb_write == PNCIO_HINT_DISABLE) { - /* romio_cb_write overrides no_indep_rw */ + GET_INFO_STR(romio_cb_write); + if (fd->hints->romio_cb_write == PNCIO_HINT_DISABLE) { + /* romio_cb_write overrides romio_no_indep_rw */ MPI_Info_set(fd->info, "romio_no_indep_rw", "false"); - fd->hints->no_indep_rw = PNCIO_HINT_DISABLE; + fd->hints->romio_no_indep_rw = PNCIO_HINT_DISABLE; } /* user intends to call collective I/O APIs only */ - Info_check_and_install_true(fd, users_info, "romio_no_indep_rw", - &fd->hints->no_indep_rw); - if (fd->hints->no_indep_rw == 1) { - /* if 'no_indep_rw' set, also hint that we will do - * collective buffering: if we aren't doing independent io, - * then we have to do collective */ + GET_INFO_STR(romio_no_indep_rw); + if (fd->hints->romio_no_indep_rw == PNCIO_HINT_ENABLE) { MPI_Info_set(fd->info, "romio_cb_write", "enable"); MPI_Info_set(fd->info, "romio_cb_read", "enable"); - fd->hints->cb_read = PNCIO_HINT_ENABLE; - fd->hints->cb_write = PNCIO_HINT_ENABLE; + fd->hints->romio_cb_read = PNCIO_HINT_ENABLE; + fd->hints->romio_cb_write = PNCIO_HINT_ENABLE; } /* enable/disable data sieving */ - Info_check_and_install_enabled(fd, users_info, "romio_ds_read", - &fd->hints->ds_read); - Info_check_and_install_enabled(fd, users_info, "romio_ds_write", - &fd->hints->ds_write); + GET_INFO_STR(romio_ds_read); + GET_INFO_STR(romio_ds_write); /* number of I/O aggregators */ - Info_check_and_install_int(fd, users_info, "cb_nodes", - &fd->hints->cb_nodes); + GET_INFO_INT(cb_nodes); /* check ill value */ if (fd->hints->cb_nodes > 0 && fd->hints->cb_nodes <= nprocs) { snprintf(value, MPI_MAX_INFO_VAL + 1, "%d", fd->hints->cb_nodes); @@ -297,30 +227,40 @@ PNCIO_File_SetInfo(PNCIO_File *fd, MPI_Info_set(fd->info, "cb_nodes", "0"); } - Info_check_and_install_int(fd, users_info, "ind_wr_buffer_size", - &fd->hints->ind_wr_buffer_size); - Info_check_and_install_int(fd, users_info, "ind_rd_buffer_size", - &fd->hints->ind_rd_buffer_size); + GET_INFO_INT(ind_wr_buffer_size); + GET_INFO_INT(ind_rd_buffer_size); /* file striping configuration */ - Info_check_and_install_int(fd, users_info, "striping_unit", - &fd->hints->striping_unit); - - Info_check_and_install_int(fd, users_info, "striping_factor", - &fd->hints->striping_factor); - - Info_check_and_install_int(fd, users_info, "start_iodevice", - &fd->hints->start_iodevice); + GET_INFO_INT(striping_unit); + GET_INFO_INT(striping_factor); + GET_INFO_INT(start_iodevice); /* Lustre overstriping ratio. 0 or 1 means disabled */ - Info_check_and_install_int(fd, users_info, "lustre_overstriping_ratio", - &fd->hints->fs_hints.lustre.overstriping_ratio); + GET_INFO_INT(lustre_overstriping_ratio); + + /* Check hint consistency among all processes */ + err = hint_consistency_check(fd); /* PnetCDF ignores the following hints. * cb_config_list * deferred_open */ - return NC_NOERR; + return err; +} + +/*----< PNCIO_File_get_info() >-----------------------------------------------*/ +int PNCIO_File_get_info(PNCIO_File *fd, + MPI_Info *info_used) +{ + int err; + + err = MPI_Info_dup(fd->info, info_used); + if (err == MPI_SUCCESS) + err = NC_NOERR; + else + err = ncmpii_error_mpi2nc(err, "MPI_Info_dup"); + + return err; } From 831402ef96bdce6fb9548edb022cef8c42812029 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 8 Nov 2025 20:17:05 -0600 Subject: [PATCH 04/19] move setting is_open inside of open call --- src/drivers/pncio/pncio_close.c | 8 +++-- src/drivers/pncio/pncio_lustre_open.c | 4 +++ src/drivers/pncio/pncio_open.c | 42 +++++++++++++++++---------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/drivers/pncio/pncio_close.c b/src/drivers/pncio/pncio_close.c index 4ecc09cc6c..c62240f74c 100644 --- a/src/drivers/pncio/pncio_close.c +++ b/src/drivers/pncio/pncio_close.c @@ -22,9 +22,11 @@ int PNCIO_File_close(PNCIO_File *fh) { int err = NC_NOERR; - err = close(fh->fd_sys); - if (err != 0) - err = ncmpii_error_posix2nc("close"); + if (fh->is_open) { + err = close(fh->fd_sys); + if (err != 0) + err = ncmpii_error_posix2nc("close"); + } if (fh->hints->ranklist != NULL) NCI_Free(fh->hints->ranklist); diff --git a/src/drivers/pncio/pncio_lustre_open.c b/src/drivers/pncio/pncio_lustre_open.c index 687b3e5910..0cd3487277 100644 --- a/src/drivers/pncio/pncio_lustre_open.c +++ b/src/drivers/pncio/pncio_lustre_open.c @@ -973,6 +973,7 @@ assert(mpi_io_mode & MPI_MODE_CREATE); err = ncmpii_error_posix2nc("Lustre set striping"); goto err_out; } + fd->is_open = 1; /* Obtain Lustre file striping parameters actually set. */ numOSTs = get_striping(fd->fd_sys, fd->filename, &pattern, @@ -993,6 +994,7 @@ assert(mpi_io_mode & MPI_MODE_CREATE); err = ncmpii_error_posix2nc("open"); goto err_out; } + fd->is_open = 1; char *env_str = getenv("MIMIC_STRIPE_SIZE"); if (env_str != NULL) @@ -1028,6 +1030,7 @@ assert(mpi_io_mode & MPI_MODE_CREATE); __FILE__,__LINE__, rank, fd->filename, strerror(errno)); return ncmpii_error_posix2nc("ioctl"); } + fd->is_open = 1; } /* construct cb_nodes rank list */ @@ -1080,6 +1083,7 @@ static int wkl=0; if (wkl == 0 && rank == 0) { printf("\nxxxx %s at %d: %s ---- err = ncmpii_error_posix2nc("open"); goto err_out; } + fd->is_open = 1; /* Only root obtains the striping information and bcast to all other * processes. diff --git a/src/drivers/pncio/pncio_open.c b/src/drivers/pncio/pncio_open.c index 38981b2c88..9c279b0ce2 100644 --- a/src/drivers/pncio/pncio_open.c +++ b/src/drivers/pncio/pncio_open.c @@ -139,6 +139,7 @@ if (rank == 0) { printf("\nxxxx %s at %d: ---- %s\n",__func__,__LINE__,fd->filen err = ncmpii_error_posix2nc("open"); goto err_out; } + fd->is_open = 1; err_out: MPI_Bcast(stripin_info, 4, MPI_INT, 0, fd->comm); @@ -154,6 +155,7 @@ if (rank == 0) { printf("\nxxxx %s at %d: ---- %s\n",__func__,__LINE__,fd->filen __func__,__LINE__, rank, fd->filename, strerror(errno)); return ncmpii_error_posix2nc("ioctl"); } + fd->is_open = 1; } /* construct cb_nodes rank list */ @@ -196,6 +198,7 @@ if (rank == 0) { printf("\nxxxx %s at %d: ---- %s\n",__func__,__LINE__,fd->filen err = ncmpii_error_posix2nc("open"); goto err_out; } + fd->is_open = 1; /* Only root obtains the striping information and bcast to all other * processes. @@ -233,7 +236,7 @@ int PNCIO_File_open(MPI_Comm comm, * called to check the file system type. */ char value[MPI_MAX_INFO_VAL + 1], int_str[16]; - int i, err, min_err; + int i, err, min_err, status=NC_NOERR; fd->comm = comm; fd->filename = filename; /* without file system type name prefix */ @@ -256,9 +259,13 @@ int PNCIO_File_open(MPI_Comm comm, else MPI_Info_dup(info, &fd->info); - err = PNCIO_File_SetInfo(fd, fd->info); - if (err != NC_NOERR) - return err; + status = PNCIO_File_SetInfo(fd, fd->info); + if (status != NC_NOERR && status != NC_EMULTIDEFINE_HINTS) { + /* Inconsistent I/O hints is not a fatal error. + * In PNCIO_File_SetInfo(), root's hints overwrite local's. + */ + goto err_out; + } #if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) for (i=0; ifile_system != PNCIO_FSTYPE_MPIIO); + /* TODO: When hint romio_no_indep_rw hint is set to true, only aggregators open + * the file. + * Note because fd->is_agg is set at the end of create/open call. + */ if (fd->file_system == PNCIO_LUSTRE) { if (amode & MPI_MODE_CREATE) err = PNCIO_Lustre_create(fd, amode); @@ -281,11 +292,10 @@ int PNCIO_File_open(MPI_Comm comm, else err = GEN_open(fd); } - if (err != NC_NOERR) goto err_out; - - /* TODO: when hint no_indep_rw hint is set to true, only aggregators open - * the file */ - fd->is_open = 1; + if (err != NC_NOERR) { /* fatal error */ + status = err; + goto err_out; + } /* set file striping hints */ snprintf(int_str, 16, "%d", fd->hints->striping_unit); @@ -323,15 +333,17 @@ int PNCIO_File_open(MPI_Comm comm, /* collective buffer is used only by I/O aggregators only */ if (fd->is_agg) { fd->io_buf = NCI_Calloc(1, fd->hints->cb_buffer_size); - if (fd->io_buf == NULL) - return NC_ENOMEM; + if (fd->io_buf == NULL) /* fatal error */ + status = NC_ENOMEM; } err_out: - MPI_Allreduce(&err, &min_err, 1, MPI_INT, MPI_MIN, comm); + MPI_Allreduce(&status, &min_err, 1, MPI_INT, MPI_MIN, comm); /* All NC errors are < 0 */ - if (min_err < 0) { - if (err == 0) /* close file if opened successfully */ + + if (min_err != NC_NOERR) { + if (status == NC_NOERR && fd->is_open) + /* close file if opened successfully */ close(fd->fd_sys); NCI_Free(fd->hints); if (fd->info != MPI_INFO_NULL) @@ -339,6 +351,6 @@ int PNCIO_File_open(MPI_Comm comm, if (fd->io_buf != NULL) NCI_Free(fd->io_buf); } - return err; + return status; } From 9d678d847a41b8f595f8248f2e661f2583b414d2 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sun, 9 Nov 2025 11:27:55 -0600 Subject: [PATCH 05/19] parallel_run.sh run ncmpidiff based on driver mode --- benchmarks/C/parallel_run.sh | 18 +++++++++++------- benchmarks/FLASH-IO/parallel_run.sh | 18 +++++++++++------- benchmarks/WRF-IO/parallel_run.sh | 18 +++++++++++------- examples/parallel_run.sh | 8 ++++---- test/parallel_run.sh | 16 ++++++++-------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/benchmarks/C/parallel_run.sh b/benchmarks/C/parallel_run.sh index 8f25930303..3ab232b8c5 100755 --- a/benchmarks/C/parallel_run.sh +++ b/benchmarks/C/parallel_run.sh @@ -32,6 +32,8 @@ unset PNETCDF_HINTS fixed_length=23 +TEST_MPIIO_MODES="0 1" + for i in ${check_PROGRAMS} ; do for j in ${safe_modes} ; do @@ -44,7 +46,7 @@ for i in ${check_PROGRAMS} ; do fi OUT_PREFIX="${TESTOUTDIR}/$i" - for mpiio_mode in 0 1 ; do + for mpiio_mode in ${TEST_MPIIO_MODES} ; do if test "$mpiio_mode" = 1 ; then USEMPIO_HINTS="nc_pncio=disable" DRIVER_OUT_FILE="${OUT_PREFIX}.mpio" @@ -128,12 +130,14 @@ for i in ${check_PROGRAMS} ; do done # mpiio_mode DIFF_OPT="-q" - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc + if test "x$TEST_MPIIO_MODES" = "x0 1" ; then + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc + fi + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc done # safe_modes rm -f ${OUTDIR}/$i*nc* diff --git a/benchmarks/FLASH-IO/parallel_run.sh b/benchmarks/FLASH-IO/parallel_run.sh index d642ff000e..1a705ee867 100755 --- a/benchmarks/FLASH-IO/parallel_run.sh +++ b/benchmarks/FLASH-IO/parallel_run.sh @@ -34,6 +34,8 @@ FILE_EXTS="ncmpi_chk_0000 ncmpi_plt_cnt_0000 ncmpi_plt_crn_0000" fixed_length=23 +TEST_MPIIO_MODES="0 1" + for i in ${check_PROGRAMS} ; do for j in ${safe_modes} ; do @@ -46,7 +48,7 @@ for i in ${check_PROGRAMS} ; do fi OUT_PREFIX="${TESTOUTDIR}/$i" - for mpiio_mode in 0 1 ; do + for mpiio_mode in ${TEST_MPIIO_MODES} ; do if test "$mpiio_mode" = 1 ; then USEMPIO_HINTS="nc_pncio=disable" DRIVER_OUT_FILE="${OUT_PREFIX}.mpio" @@ -134,12 +136,14 @@ for i in ${check_PROGRAMS} ; do done # mpiio_mode for ext in $FILE_EXTS ; do - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc + if test "x$TEST_MPIIO_MODES" = "x0 1" ; then + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc ---" + $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc ---" + $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc + fi + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc ---" + $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc done # ext done # safe_modes diff --git a/benchmarks/WRF-IO/parallel_run.sh b/benchmarks/WRF-IO/parallel_run.sh index 5f5f0a1ca0..c532411c4d 100755 --- a/benchmarks/WRF-IO/parallel_run.sh +++ b/benchmarks/WRF-IO/parallel_run.sh @@ -32,6 +32,8 @@ unset PNETCDF_HINTS fixed_length=23 +TEST_MPIIO_MODES="0 1" + for i in ${check_PROGRAMS} ; do for j in ${safe_modes} ; do @@ -44,7 +46,7 @@ for i in ${check_PROGRAMS} ; do fi OUT_PREFIX="${TESTOUTDIR}/$i" - for mpiio_mode in 0 1 ; do + for mpiio_mode in ${TEST_MPIIO_MODES} ; do if test "$mpiio_mode" = 1 ; then USEMPIO_HINTS="nc_pncio=disable" DRIVER_OUT_FILE="${OUT_PREFIX}.mpio" @@ -124,12 +126,14 @@ for i in ${check_PROGRAMS} ; do done # mpiio_mode DIFF_OPT="-q" - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc + if test "x$TEST_MPIIO_MODES" = "x0 1" ; then + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc + fi + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc done # safe_modes rm -f ${OUTDIR}/$i*nc* diff --git a/examples/parallel_run.sh b/examples/parallel_run.sh index f9771f7439..53a773de51 100755 --- a/examples/parallel_run.sh +++ b/examples/parallel_run.sh @@ -194,16 +194,16 @@ for i in ${check_PROGRAMS} ; do $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j - # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j + # echo "--- ncmpidiff $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j done else # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc + # echo "--- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc fi done # safe_modes diff --git a/test/parallel_run.sh b/test/parallel_run.sh index 694cea504a..295ea8ca78 100755 --- a/test/parallel_run.sh +++ b/test/parallel_run.sh @@ -258,8 +258,8 @@ for i in ${check_PROGRAMS} ; do $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.mpio.ina.nc$j # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.nc$j ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.nc$j - # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.ina.nc$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.ina.nc$j + # echo "--- ncmpidiff $OUT_PREFIX.pncio.nc$j $OUT_PREFIX.pncio.ina.nc$j ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc$j $OUT_PREFIX.pncio.ina.nc$j done elif test "$i" = tst_pthread ; then for j in `seq 0 ${NTHREADS}` ; do @@ -267,8 +267,8 @@ for i in ${check_PROGRAMS} ; do $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j - # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j + # echo "--- ncmpidiff $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j done elif test "$i" = mcoll_perf ; then for j in `seq 0 9` ; do @@ -277,16 +277,16 @@ for i in ${check_PROGRAMS} ; do $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.mpio.ina.$ext # echo "--- ncmpidiff $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.$ext ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.$ext - # echo "--- ncmpidiff $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.ina.$ext ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.ina.$ext + # echo "--- ncmpidiff $OUT_PREFIX.pncio.$ext $OUT_PREFIX.pncio.ina.$ext ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.$ext $OUT_PREFIX.pncio.ina.$ext done else # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "--- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.ina.nc + # echo "--- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" + $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc fi done # safe_modes From c5b785376b2775d72a1a55dd6651aeb1e624746b Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:03:54 -0600 Subject: [PATCH 06/19] make ncmpi_close() returns the same error code --- src/drivers/ncmpio/ncmpio_close.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/drivers/ncmpio/ncmpio_close.c b/src/drivers/ncmpio/ncmpio_close.c index cf8553c638..6f9cf561c9 100644 --- a/src/drivers/ncmpio/ncmpio_close.c +++ b/src/drivers/ncmpio/ncmpio_close.c @@ -249,6 +249,10 @@ ncmpio_close(void *ncdp) if (ncp->ina_comm != MPI_COMM_NULL) MPI_Comm_free(&ncp->ina_comm); + /* collectively return the same error code */ + if (ncp->nprocs > 1) + MPI_Allreduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MIN, ncp->comm); + /* free up space occupied by the header metadata */ ncmpio_free_NC(ncp); From 726f693c83612e5a760ddd5f9eace60a90da5ae6 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:07:06 -0600 Subject: [PATCH 07/19] duplicate MPI info for non-INA aggregators in case later switch to indep mode --- src/drivers/ncmpio/ncmpio_create.c | 3 ++- src/drivers/ncmpio/ncmpio_open.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_create.c b/src/drivers/ncmpio/ncmpio_create.c index b1444d0b81..5b78ffb696 100644 --- a/src/drivers/ncmpio/ncmpio_create.c +++ b/src/drivers/ncmpio/ncmpio_create.c @@ -403,7 +403,8 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == * non-aggregators can skip. */ if (comm == MPI_COMM_NULL) { - MPI_Info_create(&ncp->mpiinfo); + if (user_info != MPI_INFO_NULL) + MPI_Info_dup(user_info, &ncp->mpiinfo); goto fn_exit; } } diff --git a/src/drivers/ncmpio/ncmpio_open.c b/src/drivers/ncmpio/ncmpio_open.c index 7d5ec6d23f..f244183e1b 100644 --- a/src/drivers/ncmpio/ncmpio_open.c +++ b/src/drivers/ncmpio/ncmpio_open.c @@ -72,6 +72,7 @@ ncmpio_open(MPI_Comm comm, ncp->comm = comm; /* reuse comm duplicated in dispatch layer */ ncp->rank = rank; ncp->nprocs = nprocs; + ncp->mpiinfo = MPI_INFO_NULL; /* Extract hints from user_info. Two hints must be extracted now in order * to continue: @@ -196,7 +197,8 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == * non-aggregators can skip. */ if (comm == MPI_COMM_NULL) { - MPI_Info_create(&ncp->mpiinfo); + if (user_info != MPI_INFO_NULL) + MPI_Info_dup(user_info, &ncp->mpiinfo); goto fn_exit; } } From a00ae89d3c34ef8a4710b19027151b5351151a70 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:10:14 -0600 Subject: [PATCH 08/19] pncio_write_str_naive.c assert disp and offset are 0s --- src/drivers/pncio/pncio_write_str_naive.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/drivers/pncio/pncio_write_str_naive.c b/src/drivers/pncio/pncio_write_str_naive.c index eeed6cf474..9ac11de2bc 100644 --- a/src/drivers/pncio/pncio_write_str_naive.c +++ b/src/drivers/pncio/pncio_write_str_naive.c @@ -9,10 +9,10 @@ #include -MPI_Offset PNCIO_GEN_WriteStrided_naive(PNCIO_File *fd, - const void *buf, +MPI_Offset PNCIO_GEN_WriteStrided_naive(PNCIO_File *fd, + const void *buf, PNCIO_View buf_view, - MPI_Offset offset) + MPI_Offset offset) { int b_index; MPI_Count bufsize; @@ -80,6 +80,10 @@ assert(fd->filetype == MPI_BYTE); disp = fd->disp; +/* noncontiguous in fileview, disp and offset should be 0 for PnetCDF */ +assert(fd->disp == 0); +assert(offset == 0); + size_in_filetype = offset; sum = 0; From 64bb6f700f2826abf08996bbaeabe0fc689bb6a5 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:11:41 -0600 Subject: [PATCH 09/19] benchmarks/C/aggregation.c add indep I/O mode option --- benchmarks/C/aggregation.c | 118 ++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 33 deletions(-) diff --git a/benchmarks/C/aggregation.c b/benchmarks/C/aggregation.c index 3f2f3afccc..c834d562f4 100644 --- a/benchmarks/C/aggregation.c +++ b/benchmarks/C/aggregation.c @@ -122,6 +122,7 @@ typedef struct { int star_block; int blocking_io; int double_xtype; + int indep_io; MPI_Offset len; MPI_Offset w_size; MPI_Offset r_size; @@ -287,6 +288,11 @@ int benchmark_write(char *filename, sts = (int*) malloc(sizeof(int) * num_reqs); err = ncmpi_enddef(ncid); ERR(err) + if (cfg->indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + err = ncmpi_inq_header_size(ncid, &cfg->header_size); ERR(err) err = ncmpi_inq_header_extent(ncid, &cfg->header_extent); ERR(err) end_t = MPI_Wtime(); @@ -306,12 +312,17 @@ int benchmark_write(char *filename, start[1] = cfg->len * (rank / psizes[1]); count[1] = cfg->len; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, - buf[v], &reqs[k++]); + buf[v], &reqs[k++]); ERR(err) if (debug) DBG_PRINT("block-block", n, i); v++; @@ -323,9 +334,14 @@ int benchmark_write(char *filename, count[2] = cfg->len; stride[1] = 1; stride[2] = nprocs; - if (cfg->blocking_io) - err = ncmpi_put_vars_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vars_double(ncid, varid[v], start, count, stride, buf[v]); + else + err = ncmpi_put_vars_double_all(ncid, varid[v], start, + count, stride, buf[v]); + } else err = ncmpi_iput_vars_double(ncid, varid[v], start, count, stride, buf[v], &reqs[k++]); @@ -338,9 +354,14 @@ int benchmark_write(char *filename, start[2] = 0; count[1] = cfg->len; count[2] = bs_gsizes[2]; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, buf[v], &reqs[k++]); @@ -353,9 +374,14 @@ int benchmark_write(char *filename, start[2] = cfg->len * rank; count[1] = sb_gsizes[1]; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, buf[v], &reqs[k++]); @@ -372,13 +398,13 @@ int benchmark_write(char *filename, if (!cfg->blocking_io) { start_t = end_t; -#ifdef USE_INDEP_MODE - err = ncmpi_begin_indep_data(ncid); ERR(err) - err = ncmpi_wait(ncid, num_reqs, reqs, sts); ERR(err) - err = ncmpi_end_indep_data(ncid); ERR(err) -#else - err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); ERR(err) -#endif + + if (cfg->indep_io) + err = ncmpi_wait(ncid, num_reqs, reqs, sts); + else + err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); + ERR(err) + /* check status of all requests */ for (i=0; iindep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + /* Note that PnetCDF read the file in chunks of size 256KB, thus the read * amount may be more than the file header size */ @@ -518,8 +549,12 @@ int benchmark_read(char *filename, start[1] = cfg->len * (rank / psizes[1]); count[1] = cfg->len; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -533,9 +568,14 @@ int benchmark_read(char *filename, count[2] = cfg->len; stride[1] = 1; stride[2] = nprocs; - if (cfg->blocking_io) - err = ncmpi_get_vars_double_all(ncid, v, start, count, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vars_double(ncid, v, start, count, stride, buf[v]); + else + err = ncmpi_get_vars_double_all(ncid, v, start, count, + stride, buf[v]); + } else err = ncmpi_iget_vars_double(ncid, v, start, count, stride, buf[v], &reqs[k++]); @@ -547,8 +587,12 @@ int benchmark_read(char *filename, start[2] = 0; count[1] = cfg->len; count[2] = bs_gsizes[2]; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -560,8 +604,12 @@ int benchmark_read(char *filename, start[2] = cfg->len * rank; count[1] = sb_gsizes[1]; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -578,13 +626,12 @@ int benchmark_read(char *filename, if (!cfg->blocking_io) { start_t = end_t; -#ifdef USE_INDEP_MODE - err = ncmpi_begin_indep_data(ncid); ERR(err) - err = ncmpi_wait(ncid, num_reqs, reqs, sts); ERR(err) - err = ncmpi_end_indep_data(ncid); ERR(err) -#else - err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); ERR(err) -#endif + if (cfg->indep_io) + err = ncmpi_wait(ncid, num_reqs, reqs, sts); + else + err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); + ERR(err) + /* check status of all requests */ for (i=0; i Date: Fri, 14 Nov 2025 22:12:26 -0600 Subject: [PATCH 10/19] benchmarks/C/pnetcdf_put_vara.c add indep I/O mode option --- benchmarks/C/pnetcdf_put_vara.c | 34 +++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/benchmarks/C/pnetcdf_put_vara.c b/benchmarks/C/pnetcdf_put_vara.c index 0a49a5446a..d3fbc800cc 100644 --- a/benchmarks/C/pnetcdf_put_vara.c +++ b/benchmarks/C/pnetcdf_put_vara.c @@ -93,7 +93,8 @@ pnetcdf_io(char *filename, int kind, int len, int nvars, - int ntimes) + int ntimes, + int indep_io) { char name[128], str_att[128], *file_kind; int i, j, rank, nprocs, err, nerrs=0; @@ -175,6 +176,11 @@ pnetcdf_io(char *filename, /* exit define mode */ err = ncmpi_enddef(ncid); ERR + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR + } + /* set up the access pattern */ start[0] = 0; start[1] = len * (rank / psizes[1]); @@ -190,14 +196,22 @@ pnetcdf_io(char *filename, if (nonblocking) err = ncmpi_iput_vara_float(ncid, varid[i], start, count, buf[i], NULL); - else - err = ncmpi_put_vara_float_all(ncid, varid[i], start, + else { + if (indep_io) + err = ncmpi_put_vara_float(ncid, varid[i], start, count, buf[i]); + else + err = ncmpi_put_vara_float_all(ncid, varid[i], start, + count, buf[i]); + } ERR } } if (nonblocking) { - err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + if (indep_io) + err = ncmpi_wait(ncid, NC_REQ_ALL, NULL, NULL); + else + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); ERR } timing[1] = MPI_Wtime() - timing[1]; @@ -246,9 +260,10 @@ static void usage(char *argv0) { char *help = - "Usage: %s [-h | -q | -i | -k num |-l num | -n num | -t num] [file_name]\n" + "Usage: %s [-h | -q | -a | -i | -k num |-l num | -n num | -t num] [file_name]\n" " [-h] Print help\n" " [-q] Quiet mode (reports when fail)\n" + " [-a] Use PnetCDF independent APIs\n" " [-i] Use PnetCDF nonblocking APIs\n" " [-k format] file format: 1 for CDF-1,\n" " 2 for CDF-2,\n" @@ -266,7 +281,7 @@ int main(int argc, char** argv) extern char *optarg; char filename[512]; int i, rank, kind, nerrs=0; - int nonblocking, nvars, len, ntimes; + int nonblocking, nvars, len, ntimes, indep_io; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -277,12 +292,15 @@ int main(int argc, char** argv) nvars = 4; len = 32; ntimes = 1; + indep_io = 0; /* get command-line arguments */ - while ((i = getopt(argc, argv, "hqik:l:n:t:")) != EOF) + while ((i = getopt(argc, argv, "hqaik:l:n:t:")) != EOF) switch(i) { case 'q': verbose = 0; break; + case 'a': indep_io = 1; + break; case 'i': nonblocking = 1; break; case 'k': kind = atoi(optarg); @@ -303,7 +321,7 @@ int main(int argc, char** argv) MPI_Bcast(filename, 512, MPI_CHAR, 0, MPI_COMM_WORLD); - nerrs += pnetcdf_io(filename, nonblocking, kind, len, nvars, ntimes); + nerrs += pnetcdf_io(filename, nonblocking, kind, len, nvars, ntimes, indep_io); MPI_Finalize(); return (nerrs > 0); From 80141cdf333a68a7573536cf3d4d56da02bdda95 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:12:38 -0600 Subject: [PATCH 11/19] benchmarks/C/write_block_read_column.c add indep I/O mode option --- benchmarks/C/write_block_read_column.c | 67 +++++++++++++++++++++----- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/benchmarks/C/write_block_read_column.c b/benchmarks/C/write_block_read_column.c index 5048783f4d..9824b99b8f 100644 --- a/benchmarks/C/write_block_read_column.c +++ b/benchmarks/C/write_block_read_column.c @@ -68,6 +68,7 @@ void print_info(MPI_Info *info_used) /*----< benchmark_write() >---------------------------------------------------*/ static int benchmark_write(char *filename, + int indep_io, MPI_Offset len, MPI_Offset *w_size, MPI_Info *w_info_used, @@ -157,6 +158,12 @@ int benchmark_write(char *filename, } err = ncmpi_enddef(ncid); ERR(err) + + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + end_t = MPI_Wtime(); timing[2] = end_t - start_t; start_t = end_t; @@ -171,19 +178,31 @@ int benchmark_write(char *filename, for (i=0; i---------------------------------------------------*/ static int benchmark_read(char *filename, + int indep_io, MPI_Offset len, MPI_Offset *r_size, MPI_Info *r_info_used, @@ -241,6 +261,11 @@ int benchmark_read(char *filename, timing[1] = start_t - timing[0]; MPI_Info_free(&info); + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + err = ncmpi_inq_nvars(ncid, &nvars); ERR(err) err = ncmpi_inq_dimid(ncid, "Y", &dimid[0]); ERR(err) err = ncmpi_inq_dimid(ncid, "X", &dimid[1]); ERR(err) @@ -283,19 +308,31 @@ int benchmark_read(char *filename, for (i=0; i Date: Fri, 14 Nov 2025 22:16:33 -0600 Subject: [PATCH 12/19] benchmarks/C make ptest problem size larger --- benchmarks/C/parallel_run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/C/parallel_run.sh b/benchmarks/C/parallel_run.sh index 3ab232b8c5..863cc614d2 100755 --- a/benchmarks/C/parallel_run.sh +++ b/benchmarks/C/parallel_run.sh @@ -85,9 +85,9 @@ for i in ${check_PROGRAMS} ; do export PNETCDF_SAFE_MODE=$j # echo "PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" - CMD_OPTS="-q -l 10" + CMD_OPTS="-q -l 500" if test "$i" = "aggregation" ; then - CMD_OPTS+=" -b -c -i -j" + CMD_OPTS+=" -b -a -x -n 12 -t 2" fi # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPTS ${OUT_FILE}.nc" From 6889bbc834bd0072dc298f100fffec4d58cac011 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:17:56 -0600 Subject: [PATCH 13/19] FLASH-IO/parallel_run.sh separate cases for running ncmpidiff --- benchmarks/FLASH-IO/parallel_run.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/benchmarks/FLASH-IO/parallel_run.sh b/benchmarks/FLASH-IO/parallel_run.sh index 1a705ee867..a7a72a67f8 100755 --- a/benchmarks/FLASH-IO/parallel_run.sh +++ b/benchmarks/FLASH-IO/parallel_run.sh @@ -136,14 +136,20 @@ for i in ${check_PROGRAMS} ; do done # mpiio_mode for ext in $FILE_EXTS ; do - if test "x$TEST_MPIIO_MODES" = "x0 1" ; then + if test "x$TEST_MPIIO_MODES" = "x0" ; then + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc ---" + $NCMPIDIFF -q $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc + elif test "x$TEST_MPIIO_MODES" = "x1" ; then # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc + $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc + elif test "x$TEST_MPIIO_MODES" = "x0 1" ; then + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc ---" + $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.mpio.ina.$ext.nc # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc + $NCMPIDIFF -q $OUT_PREFIX.mpio.$ext.nc $OUT_PREFIX.pncio.$ext.nc + # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc ---" + $NCMPIDIFF -q $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc fi - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc ---" - $MPIRUN $NCMPIDIFF -q $OUT_PREFIX.pncio.$ext.nc $OUT_PREFIX.pncio.ina.$ext.nc done # ext done # safe_modes From 66764a15636df8a912b3844aea4a6c0e1aff1e0c Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:36:09 -0600 Subject: [PATCH 14/19] test/C add an option to run in indep I/O mode --- test/C/pres_temp_4D.h | 2 +- test/C/pres_temp_4D_rd.c | 67 ++++++++++++++++++++++++++++--------- test/C/pres_temp_4D_wr.c | 72 +++++++++++++++++++++++++++++++--------- test/parallel_run.sh | 7 +++- 4 files changed, 114 insertions(+), 34 deletions(-) diff --git a/test/C/pres_temp_4D.h b/test/C/pres_temp_4D.h index f70677c069..80e1a882f6 100644 --- a/test/C/pres_temp_4D.h +++ b/test/C/pres_temp_4D.h @@ -8,7 +8,7 @@ #define NREC 2 #define REC_NAME "time" #define LVL_NAME "level" -#define NLVL 10 +#define NLVL 512 /* Names of things. */ #define PRES_NAME "pressure" diff --git a/test/C/pres_temp_4D_rd.c b/test/C/pres_temp_4D_rd.c index 76078d7984..1713f95979 100644 --- a/test/C/pres_temp_4D_rd.c +++ b/test/C/pres_temp_4D_rd.c @@ -22,16 +22,32 @@ #include #include #include +#include /* getopt() */ #include /* basename() */ -#include + #include +#include #include #include "pres_temp_4D.h" +static void +usage(char *argv0) +{ + char *help = + "Usage: %s [OPTIONS]...[filename]\n" + " [-h] Print help\n" + " [-a] use independent I/O\n" + " [filename]: input netCDF file name (default: %s)\n"; + fprintf(stderr, help, argv0, FILE_NAME); +} + int main(int argc, char **argv) { - int rank, nprocs, ncid, pres_varid, temp_varid; + extern int optind; + extern char *optarg; + char filename[256]; + int rank, nprocs, ncid, pres_varid, temp_varid, indep_io; int lat_varid, lon_varid; /* The start and count arrays will tell the netCDF library where to @@ -52,20 +68,22 @@ int main(int argc, char **argv) /* Error handling. */ int err, nerrs = 0; - char *filename = FILE_NAME; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 3) { - if (!rank) - printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - if (argc > 1) filename = argv[1]; + indep_io = 0; + while ((i = getopt(argc, argv, "ha")) != EOF) + switch(i) { + case 'a': indep_io = 1; + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, FILE_NAME); + else snprintf(filename, 256, "%s", argv[optind]); /* Open the file. */ err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); @@ -75,6 +93,11 @@ int main(int argc, char **argv) goto err_out; } + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + if (rank == 0) { char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); int format; @@ -96,9 +119,15 @@ int main(int argc, char **argv) /* Read the coordinate variable data. */ memset(lats, 0, sizeof(float) * NLAT); memset(lons, 0, sizeof(float) * NLON); - err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]); + if (indep_io) + err = ncmpi_get_var_float(ncid, lat_varid, &lats[0]); + else + err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]); CHECK_ERR - err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]); + if (indep_io) + err = ncmpi_get_var_float(ncid, lat_varid, &lats[0]); + else + err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]); CHECK_ERR /* Check the coordinate variable data. */ @@ -164,9 +193,15 @@ int main(int argc, char **argv) /* Read and check one record at a time. */ for (rec = 0; rec < NREC; rec++) { start[0] = rec; - err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]); + if (indep_io) + err = ncmpi_get_vara_float(ncid, pres_varid, start, count, &pres_in[0][0]); + else + err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]); CHECK_ERR - err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]); + if (indep_io) + err = ncmpi_get_vara_float(ncid, temp_varid, start, count, &temp_in[0][0]); + else + err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]); CHECK_ERR /* Check the data. */ diff --git a/test/C/pres_temp_4D_wr.c b/test/C/pres_temp_4D_wr.c index 8039d0dcdb..9ea42bba82 100644 --- a/test/C/pres_temp_4D_wr.c +++ b/test/C/pres_temp_4D_wr.c @@ -22,18 +22,36 @@ #include #include #include +#include /* getopt() */ #include /* basename() */ -#include + #include +#include #include #include "pres_temp_4D.h" +static void +usage(char *argv0) +{ + char *help = + "Usage: %s [OPTIONS]...[filename]\n" + " [-h] Print help\n" + " [-a] use independent I/O\n" + " [-f num]: output file format: 1,2,5 for classic, 4 for netCDF4\n" + " [filename]: output netCDF file name (default: %s)\n"; + fprintf(stderr, help, argv0, FILE_NAME); +} + int main(int argc, char **argv) { + extern int optind; + extern char *optarg; + char filename[256]; + /* IDs for the netCDF file, dimensions, and variables. */ int nprocs, rank, nerrs = 0; - int ncid; + int ncid, indep_io, fmt; int lon_dimid, lat_dimid, lvl_dimid, rec_dimid; int lat_varid, lon_varid, pres_varid, temp_varid; int dimids[NDIMS]; @@ -57,23 +75,34 @@ int main(int argc, char **argv) /* Error handling. */ int err; - char *filename = FILE_NAME; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 3) { - if (!rank) - printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - if (argc > 1) filename = argv[1]; - - if (argc > 2 && atoi(argv[2]) == 4) + indep_io = 0; + fmt = 1; + while ((i = getopt(argc, argv, "haf:")) != EOF) + switch(i) { + case 'a': indep_io = 1; + break; + case 'f': fmt = atoi(optarg); + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, FILE_NAME); + else snprintf(filename, 256, "%s", argv[optind]); + + if (fmt == 1) + format = NC_FORMAT_CLASSIC; + else if (fmt == 2) + format = NC_FORMAT_64BIT_OFFSET; + else if (fmt == 4) format = NC_FORMAT_NETCDF4; + else if (fmt == 5) + format = NC_FORMAT_64BIT_DATA; if (rank == 0) { char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); @@ -161,6 +190,11 @@ int main(int argc, char **argv) err = ncmpi_enddef(ncid); CHECK_ERR + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + err = ncmpi_begin_indep_data(ncid); /* Write the coordinate variable data. This will put the latitudes and longitudes of our data grid into the netCDF file. */ @@ -224,9 +258,15 @@ int main(int argc, char **argv) for (rec = 0; rec < NREC; rec++) { start[0] = rec; - err = ncmpi_put_vara_float_all(ncid, pres_varid, start, count, &pres_out[0][0]); + if (indep_io) + err = ncmpi_put_vara_float(ncid, pres_varid, start, count, &pres_out[0][0]); + else + err = ncmpi_put_vara_float_all(ncid, pres_varid, start, count, &pres_out[0][0]); CHECK_ERR - err = ncmpi_put_vara_float_all(ncid, temp_varid, start, count, &temp_out[0][0]); + if (indep_io) + err = ncmpi_put_vara_float(ncid, temp_varid, start, count, &temp_out[0][0]); + else + err = ncmpi_put_vara_float_all(ncid, temp_varid, start, count, &temp_out[0][0]); CHECK_ERR } diff --git a/test/parallel_run.sh b/test/parallel_run.sh index 295ea8ca78..d39430e844 100755 --- a/test/parallel_run.sh +++ b/test/parallel_run.sh @@ -290,6 +290,11 @@ for i in ${check_PROGRAMS} ; do fi done # safe_modes - rm -f ${OUTDIR}/$i*nc* + + if test "x$i" = xpres_temp_4D_wr ; then + rm -f ${OUTDIR}/pres_temp_4D*.nc* + else + rm -f ${OUTDIR}/$i*nc* + fi done # check_PROGRAMS From 995bcac523ab3514b7f39636753bdf0273b900c4 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:37:45 -0600 Subject: [PATCH 15/19] test/nonblocking/bput_varn.m4 add an option to put multiple variables --- test/nonblocking/bput_varn.m4 | 383 ++++++++++++++++++++++++---------- 1 file changed, 267 insertions(+), 116 deletions(-) diff --git a/test/nonblocking/bput_varn.m4 b/test/nonblocking/bput_varn.m4 index 8cc4a9541c..639990c11f 100644 --- a/test/nonblocking/bput_varn.m4 +++ b/test/nonblocking/bput_varn.m4 @@ -35,28 +35,28 @@ dnl * data: * * var0 = - * 13, 13, 13, 11, 11, 10, 10, 12, 11, 11, - * 10, 12, 12, 12, 13, 11, 11, 12, 12, 12, - * 11, 11, 12, 13, 13, 13, 10, 10, 11, 11, - * 10, 10, 10, 12, 11, 11, 11, 13, 13, 13 ; + * 3, 3, 3, 1, 1, 0, 0, 2, 1, 1, + * 0, 2, 2, 2, 3, 1, 1, 2, 2, 2, + * 1, 1, 2, 3, 3, 3, 0, 0, 1, 1, + * 0, 0, 0, 2, 1, 1, 1, 3, 3, 3 ; * var1 = - * 12, 12, 12, 10, 10, 13, 13, 11, 10, 10, - * 13, 11, 11, 11, 12, 10, 10, 11, 11, 11, - * 10, 10, 11, 12, 12, 12, 13, 13, 10, 10, - * 13, 13, 13, 11, 10, 10, 10, 12, 12, 12 ; + * 2, 2, 2, 0, 0, 3, 3, 1, 0, 0, + * 3, 1, 1, 1, 2, 0, 0, 1, 1, 1, + * 0, 0, 1, 2, 2, 2, 3, 3, 0, 0, + * 3, 3, 3, 1, 0, 0, 0, 2, 2, 2 ; * * var2 = - * 11, 11, 11, 13, 13, 12, 12, 10, 13, 13, - * 12, 10, 10, 10, 11, 13, 13, 10, 10, 10, - * 13, 13, 10, 11, 11, 11, 12, 12, 13, 13, - * 12, 12, 12, 10, 13, 13, 13, 11, 11, 11 ; + * 1, 1, 1, 3, 3, 2, 2, 0, 3, 3, + * 2, 0, 0, 0, 1, 3, 3, 0, 0, 0, + * 3, 3, 0, 1, 1, 1, 2, 2, 3, 3, + * 2, 2, 2, 0, 3, 3, 3, 1, 1, 1 ; * * var3 = - * 10, 10, 10, 12, 12, 11, 11, 13, 12, 12, - * 11, 13, 13, 13, 10, 12, 12, 13, 13, 13, - * 12, 12, 13, 10, 10, 10, 11, 11, 12, 12, - * 11, 11, 11, 13, 12, 12, 12, 10, 10, 10 ; + * 0, 0, 0, 2, 2, 1, 1, 3, 2, 2, + * 1, 3, 3, 3, 0, 2, 2, 3, 3, 3, + * 2, 2, 3, 0, 0, 0, 1, 1, 2, 2, + * 1, 1, 1, 3, 2, 2, 2, 0, 0, 0 ; * } * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -68,6 +68,8 @@ dnl #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ + #include #include @@ -76,6 +78,7 @@ dnl #define NLOOPS 4 #define MAX_NREQS 6 #define NDIMS 2 +#define NVARS 4 #define NY 4 #define NX 10 @@ -92,6 +95,7 @@ include(`utils.m4')dnl printf("Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ ncmpi_strerrno((a)[_i])); \ nerrs++; \ + goto err_out; \ } \ } \ } @@ -162,70 +166,82 @@ void permute(MPI_Offset *a, MPI_Offset *b) define(`TEST_BPUT_VARN',`dnl static -int clear_file_contents_$1(int ncid, int *varid) +int clear_file_contents_$1(int ncid, int *varid, int coll_io) { - int i, err, nerrs=0, rank; + int i, err, nerrs=0; $1 *w_buffer = ($1*) malloc(sizeof($1) * NY*NX); - for (i=0; i 4) MPI_Barrier(MPI_COMM_WORLD); - for (i=0; i<4; i++) { - for (j=0; j= 10+nprocs) continue; - if (r_buffer[j] != expected[i][j]) { - printf("Expected read $1 buf[%d][%d]=%d, but got %d\n", - i,j,(int)expected[i][j],(int)r_buffer[j]); + $1 exp = (expected[i][j] >= nprocs) ? fillv : expected[i][j]; + if (r_buffer[j] != exp) { + char var_name[16]; + ncmpi_inq_varname(ncid, varid[i], var_name); + printf("Expected var %s read $1 buf[%d][%d]=%d, but got %d\n", + var_name,i,j,(int)exp,(int)r_buffer[j]); nerrs++; - goto fn_exit; + break; } } + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + if (nerrs > 0) goto fn_exit; } fn_exit: free(r_buffer); @@ -233,7 +249,7 @@ fn_exit: } static int -test_bput_varn_$1(char *filename, int cdf) +test_bput_varn_$1(char *filename, int cdf, int coll_io) { int i, j, k, rank, err, nerrs=0, bb_enabled; int ncid, cmode, varid[NLOOPS], dimid[2], nreqs, reqs[NLOOPS], sts[NLOOPS]; @@ -299,15 +315,6 @@ test_bput_varn_$1(char *filename, int cdf) MPI_Info_free(&infoused); } - /* create a global array of size NY * NX */ - err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR - err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var0", NC_TYPE($1), NDIMS, dimid, &varid[0]); CHECK_ERR - err = ncmpi_def_var(ncid, "var1", NC_TYPE($1), NDIMS, dimid, &varid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var2", NC_TYPE($1), NDIMS, dimid, &varid[2]); CHECK_ERR - err = ncmpi_def_var(ncid, "var3", NC_TYPE($1), NDIMS, dimid, &varid[3]); CHECK_ERR - err = ncmpi_enddef(ncid); CHECK_ERR - /* allocate space for starts and counts */ starts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); counts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); @@ -338,17 +345,32 @@ test_bput_varn_$1(char *filename, int cdf) } } + /* only rank 0, 1, 2, and 3 do I/O: + * each of ranks 0 to 3 write nreqs nonblocking requests */ + nreqs = 4; + if (rank >= 4) nreqs = 0; + + /* create a global array of size NY * NX */ + err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR + for (i=0; i= 4) nreqs = 0; - /* bufsize must be max of data type converted before and after */ MPI_Offset bufsize = 0; @@ -364,7 +386,7 @@ test_bput_varn_$1(char *filename, int cdf) /* allocate I/O buffer and initialize its contents */ buffer[i] = ($1*) malloc(sizeof($1) * req_lens[i]); - for (j=0; j 0) goto err_out; /* write using varn API, one bput call per variable */ for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (nerrs > 0) goto err_out; + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) - if (!bb_enabled) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; /* permute write order: so starts[*] are not in an increasing order: * swap segment 0 with segment 2 and swap segment 1 with segment 3 @@ -422,88 +469,140 @@ test_bput_varn_$1(char *filename, int cdf) } /* write using varn API, one bput call per variable */ - nerrs += clear_file_contents_$1(ncid, varid); + nerrs += clear_file_contents_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (nerrs > 0) goto err_out; + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) - if (!bb_enabled) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + /* check if write buffer contents have been altered */ for (i=0; i 0) goto err_out; - if (!bb_enabled) + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; /* permute back to original order */ for (i=0; i 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + /* check if write buffer contents have been altered */ for (i=0; i 0) goto err_out; - if (!bb_enabled) + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); +err_out: /* free the buffer space for bput */ if (bufsize > 0) { err = ncmpi_buffer_detach(ncid); CHECK_ERR @@ -593,23 +720,41 @@ TEST_BPUT_VARN(double) TEST_BPUT_VARN(longlong) TEST_BPUT_VARN(ulonglong) + +define(`TEST_DATA_TYPE',`test_bput_varn_$1(filename, cdf_formats[fmt], coll_io)') + +#define FILE_NAME "testfile.nc" + +static void +usage(char *argv0) +{ + char *help = + "Usage: %s [OPTIONS]...[filename]\n" + " [-h] Print help\n" + " [filename]: output netCDF file name (default: %s)\n"; + fprintf(stderr, help, argv0, FILE_NAME); +} + int main(int argc, char** argv) { + extern int optind; + extern char *optarg; char filename[256]; - int i, rank, err, nerrs=0; + int i, fmt, coll_io, rank, err, nerrs=0; int cdf_formats[3]={NC_FORMAT_CLASSIC, NC_FORMAT_CDF2, NC_FORMAT_CDF5}; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + while ((i = getopt(argc, argv, "h")) != EOF) + switch(i) { + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, FILE_NAME); + else snprintf(filename, 256, "%s", argv[optind]); if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); @@ -618,19 +763,24 @@ int main(int argc, char** argv) free(cmd_str); } - for (i=0; i<3; i++) { - nerrs += test_bput_varn_text(filename, cdf_formats[i]); - nerrs += test_bput_varn_schar(filename, cdf_formats[i]); - nerrs += test_bput_varn_short(filename, cdf_formats[i]); - nerrs += test_bput_varn_int(filename, cdf_formats[i]); - nerrs += test_bput_varn_float(filename, cdf_formats[i]); - nerrs += test_bput_varn_double(filename, cdf_formats[i]); - if (cdf_formats[i] == NC_FORMAT_CDF5) { - nerrs += test_bput_varn_uchar(filename, cdf_formats[i]); - nerrs += test_bput_varn_ushort(filename, cdf_formats[i]); - nerrs += test_bput_varn_uint(filename, cdf_formats[i]); - nerrs += test_bput_varn_longlong(filename, cdf_formats[i]); - nerrs += test_bput_varn_ulonglong(filename, cdf_formats[i]); + for (coll_io=0; coll_io<2; coll_io++) { + /* test both independent and collective I/O modes */ + for (fmt=0; fmt<3; fmt++) { + /* test CDF-1, CDF-2, and CDF-5 file formats */ + nerrs += TEST_DATA_TYPE(text); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(schar); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(short); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(int); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(float); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(double); if (nerrs > 0) goto err_out; + if (nerrs > 0) goto err_out; + if (cdf_formats[fmt] == NC_FORMAT_CDF5) { + nerrs += TEST_DATA_TYPE(uchar); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(ushort); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(uint); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(longlong); if (nerrs > 0) goto err_out; + nerrs += TEST_DATA_TYPE(ulonglong); if (nerrs > 0) goto err_out; + } } } @@ -645,6 +795,7 @@ int main(int argc, char** argv) if (malloc_size > 0) ncmpi_inq_malloc_list(); } +err_out: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); From 97b0c2f35d84c0357671305b53882c874ad4419b Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:39:00 -0600 Subject: [PATCH 16/19] column_wise.m4 add testing for indep I/O mode --- test/nonblocking/column_wise.m4 | 118 ++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 42 deletions(-) diff --git a/test/nonblocking/column_wise.m4 b/test/nonblocking/column_wise.m4 index c5c354c43a..dbac240ce8 100644 --- a/test/nonblocking/column_wise.m4 +++ b/test/nonblocking/column_wise.m4 @@ -25,7 +25,7 @@ * * % m4 column_wise.m4 > column_wise.c * % mpicc -O2 -o column_wise column_wise.c -lpnetcdf - * % mpiexec -l -n 4 ./column_wise /pvfs2/wkliao/testfile.nc + * % mpiexec -l -n 4 ./column_wise -l 4 /pvfs2/wkliao/testfile.nc * 0: 0: myOff= 0 myNX= 4 * 1: 1: myOff= 4 myNX= 4 * 2: 2: myOff= 8 myNX= 4 @@ -67,22 +67,26 @@ #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ + #include #include #include #define NY 10 -#define NX 4 +#define NX 70 typedef char text; include(`foreach.m4')dnl include(`utils.m4')dnl +define(`TEST_DATA_TYPE',`test_column_wise_$1(filename, cdf_formats[fmt], coll_io, len)') + define(`TEST_COLUMN_WISE',`dnl static -int test_column_wise_$1(char *filename, int cdf) +int test_column_wise_$1(char *filename, int cdf, int coll_io, int len) { int i, j, nerrs=0, rank, nprocs, err, myNX, G_NX, myOff, num_reqs; int ncid, cmode, varid, dimid[2], *reqs, *sts; @@ -100,10 +104,10 @@ int test_column_wise_$1(char *filename, int cdf) err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR - /* the global array is NY * (NX * nprocs) */ - G_NX = NX * nprocs; - myOff = NX * rank; - myNX = NX; + /* the global array is NY * (len * nprocs) */ + G_NX = len * nprocs; + myOff = len * rank; + myNX = len; err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", G_NX, &dimid[1]); CHECK_ERR @@ -124,6 +128,11 @@ int test_column_wise_$1(char *filename, int cdf) */ err = ncmpi_flush(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* initialize the buffer with rank ID. Also make the case interesting, by allocatsing buffersd separately */ for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + len = NX; + while ((i = getopt(argc, argv, "hl:")) != EOF) + switch(i) { + case 'l': len = atoi(optarg); + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, FILE_NAME); + else snprintf(filename, 256, "%s", argv[optind]); if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); @@ -319,30 +361,21 @@ int main(int argc, char** argv) free(cmd_str); } - for (i=0; i<3; i++) { - nerrs += test_column_wise_text(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_schar(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_short(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_int(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_float(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_double(filename, cdf_formats[i]); - if (nerrs > 0) break; - if (cdf_formats[i] == NC_FORMAT_CDF5) { - nerrs += test_column_wise_uchar(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_ushort(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_uint(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_longlong(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_ulonglong(filename, cdf_formats[i]); - if (nerrs > 0) break; + for (coll_io=0; coll_io<2; coll_io++) { + for (fmt=0; fmt<3; fmt++) { + nerrs = TEST_DATA_TYPE(text); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(schar); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(short); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(int); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(float); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(double); if (nerrs > 0) goto err_out; + if (cdf_formats[fmt] == NC_FORMAT_CDF5) { + nerrs = TEST_DATA_TYPE(uchar); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(ushort); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(uint); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(longlong); if (nerrs > 0) goto err_out; + nerrs = TEST_DATA_TYPE(ulonglong); if (nerrs > 0) goto err_out; + } } } @@ -357,6 +390,7 @@ int main(int argc, char** argv) if (malloc_size > 0) ncmpi_inq_malloc_list(); } +err_out: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); From b72b67f439d89b4fe616d810263a36f026b8db4c Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:40:08 -0600 Subject: [PATCH 17/19] flexible_bput.c add testing for indep I/O mode --- test/nonblocking/flexible_bput.c | 173 +++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 43 deletions(-) diff --git a/test/nonblocking/flexible_bput.c b/test/nonblocking/flexible_bput.c index 468af74b6c..d109cbec33 100644 --- a/test/nonblocking/flexible_bput.c +++ b/test/nonblocking/flexible_bput.c @@ -21,7 +21,7 @@ * * % mpicc -O2 -o flexible_bput flexible_bput.c -lpnetcdf * - * % mpiexec -l -n 4 ./flexible_bput /pvfs2/wkliao/testfile.nc + * % mpiexec -l -n 4 ./flexible_bput -l 4 /pvfs2/wkliao/testfile.nc * * % ncmpidump /pvfs2/wkliao/testfile.nc * netcdf testfile { @@ -48,14 +48,16 @@ #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ #include + #include #include #include #define NY 6 -#define NX 4 +#define NX 70 #define GHOST 2 #define INIT_PUT_BUF(buf) \ @@ -78,6 +80,7 @@ printf("Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ __LINE__,__FILE__,i,j,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ else { \ @@ -85,6 +88,7 @@ printf("Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ __LINE__,__FILE__,i,j,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ } \ @@ -101,51 +105,47 @@ if (i < GHOST || GHOST+array_of_subsizes[0] <= i || \ j < GHOST || GHOST+array_of_subsizes[1] <= j) { \ if (buf[i][j] != -2) { \ - printf("Error at line %d in %s: unexpected get buffer[%d][%d]=%f\n", \ - __LINE__,__FILE__,i,j,(double)buf[i][j]); \ + printf("Error at line %d in %s: expect buffer[%d][%d] to be %d but got %f\n", \ + __LINE__,__FILE__,i,j,-2,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ else { \ - if (buf[i][j] != (i-GHOST)*array_of_subsizes[1]+(j-GHOST)) { \ - printf("Error at line %d in %s: unexpected get buffer[%d][%d]=%f\n", \ - __LINE__,__FILE__,i,j,(double)buf[i][j]); \ + int exp = (i-GHOST)*array_of_subsizes[1]+(j-GHOST); \ + if (buf[i][j] != exp) { \ + printf("Error at line %d in %s: expect buffer[%d][%d] to be %d but got %f\n", \ + __LINE__,__FILE__,i,j,exp,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ } \ } -int main(int argc, char** argv) +static +int tst_run(const char *filename, int coll_io, int len) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0, req, status; int ncid, cmode, varid, dimid[2]; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; - int buf_int[NX+2*GHOST][NY+2*GHOST]; - double buf_dbl[NX+2*GHOST][NY+2*GHOST]; + int **buf_int; + double **buf_dbl; MPI_Offset start[2], count[2], stride[2], imap[2]; MPI_Datatype subarray; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + buf_int = (int**) malloc(sizeof(int*) * (len+2*GHOST)); + buf_int[0] = (int*) malloc(sizeof(int) * (len+2*GHOST) * (NY+2*GHOST)); + for (i=1; i<(len+2*GHOST); i++) + buf_int[i] = buf_int[i-1] + (NY+2*GHOST); - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible bput_varm ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + buf_dbl = (double**) malloc(sizeof(double*) * (len+2*GHOST)); + buf_dbl[0] = (double*) malloc(sizeof(double) * (len+2*GHOST) * (NY+2*GHOST)); + for (i=1; i<(len+2*GHOST); i++) + buf_dbl[i] = buf_dbl[i-1] + (NY+2*GHOST); /* create a new file for writing ----------------------------------------*/ cmode = NC_CLOBBER | NC_64BIT_DATA; @@ -153,22 +153,27 @@ int main(int argc, char** argv) CHECK_ERR /* define 2 dimensions */ - err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR - err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR + err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", len*nprocs, &dimid[1]); CHECK_ERR - /* define a variable of size NY * (NX * nprocs) */ + /* define a variable of size NY * (len * nprocs) */ err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR - start[0] = 0; start[1] = NX * rank; - count[0] = NY; count[1] = NX; + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + start[0] = 0; start[1] = len * rank; + count[0] = NY; count[1] = len; stride[0] = 1; stride[1] = 1; - imap[0] = 1; imap[1] = NY; /* would be {NX, 1} if not transposing */ + imap[0] = 1; imap[1] = NY; /* would be {len, 1} if not transposing */ /* var is partitioned along X dimension in a matrix transported way */ - array_of_sizes[0] = NX + 2*GHOST; + array_of_sizes[0] = len + 2*GHOST; array_of_sizes[1] = NY + 2*GHOST; - array_of_subsizes[0] = NX; + array_of_subsizes[0] = len; array_of_subsizes[1] = NY; array_of_starts[0] = GHOST; array_of_starts[1] = GHOST; @@ -184,13 +189,18 @@ int main(int argc, char** argv) for (i=0; i<2; i++) bufsize *= count[i]; err = ncmpi_buffer_attach(ncid, bufsize); CHECK_ERR - err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_int, + err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_int[0], 1, subarray, &req); CHECK_ERR /* check if the contents of put buffer are altered */ CHECK_PUT_BUF(buf_int) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + + CHECK_ERR err = status; CHECK_ERR /* check the contents of put buffer are altered */ @@ -202,11 +212,20 @@ int main(int argc, char** argv) /* initiate get buffer contents */ INIT_GET_BUF(buf_int) - /* calling a blocking flexible API */ - err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf_int, - 1, subarray); + if (!coll_io) { + /* calling a blocking flexible API */ + err = ncmpi_end_indep_data(ncid); + CHECK_ERR + } + err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, + buf_int[0], 1, subarray); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* check the contents of get buffer */ CHECK_GET_BUF(buf_int) @@ -224,14 +243,18 @@ int main(int argc, char** argv) err = ncmpi_buffer_attach(ncid, bufsize); CHECK_ERR - err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_dbl, + err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_dbl[0], 1, subarray, &req); CHECK_ERR /* check the contents of put buffer are altered */ CHECK_PUT_BUF(buf_dbl) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR /* check the contents of put buffer are altered */ @@ -244,17 +267,80 @@ int main(int argc, char** argv) INIT_GET_BUF(buf_dbl) /* calling a blocking flexible API */ - err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf_dbl, - 1, subarray); + if (coll_io) + err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, + buf_dbl[0], 1, subarray); + else + err = ncmpi_get_varm(ncid, varid, start, count, stride, imap, + buf_dbl[0], 1, subarray); CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(buf_dbl) +err_out: MPI_Type_free(&subarray); err = ncmpi_close(ncid); CHECK_ERR + free(buf_int[0]); + free(buf_int); + free(buf_dbl[0]); + free(buf_dbl); + + return nerrs; +} + +#define FILE_NAME "testfile.nc" + +static void +usage(char *argv0) +{ + char *help = + "Usage: %s [OPTIONS]...[filename]\n" + " [-h] Print help\n" + " [-l num]: X dimension size of local array\n" + " [filename]: output netCDF file name (default: %s)\n"; + fprintf(stderr, help, argv0, FILE_NAME); +} + +int main(int argc, char** argv) +{ + extern int optind; + extern char *optarg; + char filename[256]; + int i, len, rank, nprocs, nerrs=0, err=0; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + len = NX; + while ((i = getopt(argc, argv, "hl:")) != EOF) + switch(i) { + case 'l': len = atoi(optarg); + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, FILE_NAME); + else snprintf(filename, 256, "%s", argv[optind]); + + if (rank == 0) { + char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); + sprintf(cmd_str, "*** TESTING C %s for flexible bput_varm ", basename(argv[0])); + printf("%-66s ------ ", cmd_str); + free(cmd_str); + } + + nerrs = tst_run(filename, 0, len); + if (nerrs != 0) goto err_out; + + nerrs = tst_run(filename, 1, len); + if (nerrs != 0) goto err_out; + /* check if PnetCDF freed all internal malloc */ MPI_Offset malloc_size, sum_size; err = ncmpi_inq_malloc_size(&malloc_size); @@ -267,6 +353,7 @@ int main(int argc, char** argv) if (malloc_size > 0) ncmpi_inq_malloc_list(); } +err_out: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); From a532d59eeada9ef39578e68e8887a38d053738dd Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 14 Nov 2025 22:41:44 -0600 Subject: [PATCH 18/19] testcases/alignment_test.c increases problem size --- test/testcases/alignment_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testcases/alignment_test.c b/test/testcases/alignment_test.c index c04a2377cc..f78eea3ff7 100644 --- a/test/testcases/alignment_test.c +++ b/test/testcases/alignment_test.c @@ -31,7 +31,7 @@ #include #define NVARS 8 -#define NX 5 +#define NX 70 static int tst_mode(char *filename, int mode) From 4aea2f2466c8602a998d39f8ece9ac238c66b57d Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 15 Nov 2025 15:14:20 -0600 Subject: [PATCH 19/19] tst_multi_redefine.c tests entering define mode multiple times --- test/testcases/Makefile.am | 3 +- test/testcases/tst_multi_redefine.c | 255 ++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 test/testcases/tst_multi_redefine.c diff --git a/test/testcases/Makefile.am b/test/testcases/Makefile.am index 41636d39ff..7cba755a69 100644 --- a/test/testcases/Makefile.am +++ b/test/testcases/Makefile.am @@ -99,7 +99,8 @@ TESTPROGRAMS = file_create_open \ tst_del_attr \ tst_redefine \ tst_grow_header \ - tst_varn_var1 + tst_varn_var1 \ + tst_multi_redefine M4_SRCS = put_all_kinds.m4 \ erange_fill.m4 \ diff --git a/test/testcases/tst_multi_redefine.c b/test/testcases/tst_multi_redefine.c new file mode 100644 index 0000000000..e57c61980c --- /dev/null +++ b/test/testcases/tst_multi_redefine.c @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * This program tests entering define modes multiple times. + * + * The compile and run commands are given below. + * + * % mpicc -g -o tst_multi_redefine tst_multi_redefine.c -lpnetcdf + * + * % mpiexec -l -n 4 ./tst_multi_redefine testfile.nc + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include /* getopt() */ + +#include + +#include + +#define NROUNDS 2 +#define NY 10 +#define NX 10 +#define NVARS 2 + +static int verbose; + +static int +tst_fmt(char *filename, + int cmode, + int nRounds, + int len_y, + int len_x) +{ + int i, j, k, rank, nprocs, ncid, err, nerrs=0, dimids[3]; + MPI_Offset old_hdr_size, new_hdr_size; + MPI_Offset old_hdr_ext, new_hdr_ext; + + MPI_Info info=MPI_INFO_NULL; + + MPI_Comm comm = MPI_COMM_WORLD; + + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + + if (verbose && rank == 0) { + char *fmt; + if (cmode == 0) + fmt = "NC_CLASSIC"; + else if (cmode == NC_64BIT_OFFSET) + fmt = "NC_64BIT_OFFSET"; + else if (cmode == NC_64BIT_DATA) + fmt = "NC_64BIT_DATA"; + printf("\n---- Testing file format %s ----\n", fmt); + } + + /* create a new file */ + cmode |= NC_CLOBBER; + err = ncmpi_create(comm, filename, cmode, info, &ncid); CHECK_ERROUT + + err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0]); CHECK_ERROUT + err = ncmpi_def_dim(ncid, "Y", len_y, &dimids[1]); CHECK_ERROUT + err = ncmpi_def_dim(ncid, "X", len_x*nprocs, &dimids[2]); CHECK_ERROUT + + /* Default v_align of 512 is pretty bad choice, as almost every iteration + * grows the file extension. + */ + // err = ncmpi__enddef(ncid, 0, 4096, 0, 0); + err = ncmpi_enddef(ncid); + CHECK_ERROUT + + err = ncmpi_inq_header_size(ncid, &old_hdr_size); CHECK_ERROUT + err = ncmpi_inq_header_extent(ncid, &old_hdr_ext); CHECK_ERROUT + if (verbose && rank == 0) + printf("Newly created file with header size %lld, extension %lld\n", + old_hdr_size, old_hdr_ext); + + k = 0; + for (i=0; i old_hdr_ext) + printf("Iternation %d: file header extension grows from %lld to %lld\n", + i, old_hdr_ext, new_hdr_ext); + } + old_hdr_size = new_hdr_size; + old_hdr_ext = new_hdr_ext; + + /* write to new variables */ + MPI_Offset start[3], count[3]; + start[0] = 0; + start[1] = 0; + start[2] = rank * len_x; + count[0] = 1; + count[1] = len_y; + count[2] = len_x; + + /* write to fix-sized variables */ + int *int_buf = (int*) malloc(sizeof(int) * count[0] * count[1]); + for (j=0; j 0) goto err_out; + } + + /* check if PnetCDF freed all internal malloc */ + MPI_Offset malloc_size, sum_size; + err = ncmpi_inq_malloc_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", + sum_size); + if (malloc_size > 0) ncmpi_inq_malloc_list(); + } + +err_out: + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + if (rank == 0) { + if (nerrs) printf(FAIL_STR,nerrs); + else printf(PASS_STR); + } + + MPI_Finalize(); + return (nerrs > 0); +} +