PnetCDF does not support per-variable collective/independent data mod…

…e change; modify nc_var_par_access to ignore varid

docs/tutorial.dox

@@ -809,7 +809,11 @@ create/open a netCDF file with parallel access.
 
 Parallel file access is either collective (all processors must
 participate) or independent (any processor may access the data without
-waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access().
+waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective.
+To change from collective to independent mode or vis versa, call
+nc_var_par_access() with argument 'access' set to either NC_INDEPENDENT or
+NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid' is ignored, as
+PnetCDF does not support per-variable collective/independent mode change.
 
 \page tutorial_ncids Numbering of NetCDF IDs
 

examples/C/parallel_vara.c

@@ -150,7 +150,7 @@ int main(int argc, char** argv)
     err = nc_enddef(ncid); ERR
 
     /* set to use MPI/PnetCDF collective I/O */
-    err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR
+    err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR
 
     /* now we are in data mode */
     start[0] = 0;
@@ -176,7 +176,7 @@ int main(int argc, char** argv)
     err = nc_inq_varid(ncid, "var", &varid); ERR
 
     /* set to use MPI/PnetCDF collective I/O */
-    err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR
+    err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR
 
     /* each process reads its subarray from the file */
     err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR

libsrcp/ncpdispatch.c

@@ -9,10 +9,14 @@
 #include <mpi.h>
 #include "nc.h"
 #include "ncdispatch.h"
+#include "fbits.h"
 
 /* Must follow netcdf.h */
 #include <pnetcdf.h>
 
+#define NCP_MODE_DATA  0x0001
+#define NCP_MODE_INDEP 0x0002
+
 typedef struct NCP_INFO
 {
    /* pnetcdf_file will be true if the file is created/opened with the
@@ -104,6 +108,9 @@ NCP_create(const char *path, int cmode,
 
     res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid));
 
+    /* Default to independent access, like netCDF-4/HDF5 files. */
+    fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
+
     if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */
 done:
     return res;
@@ -152,6 +159,9 @@ NCP_open(const char *path, int cmode,
     nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO));
     if(nc5 == NULL) {res = NC_ENOMEM; goto done;}
 
+    /* file open automatically enters data mode */
+    fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
+
     /* Link nc5 and nc */
     NCP_DATA_SET(nc,nc5);
 
@@ -160,7 +170,7 @@ NCP_open(const char *path, int cmode,
     /* Default to independent access, like netCDF-4/HDF5 files. */
     if(!res) {
 	res = ncmpi_begin_indep_data(nc->int_ncid);
-	nc5->pnetcdf_access_mode = NC_INDEPENDENT;
+	fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
     }
 done:
     return res;
@@ -170,8 +180,14 @@ static int
 NCP_redef(int ncid)
 {
     NC* nc;
+    NCP_INFO* nc5;
+
     int status = NC_check_id(ncid, &nc);
     if(status != NC_NOERR) return status;
+
+    nc5 = NCP_DATA(nc);
+    fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
+
     return ncmpi_redef(nc->int_ncid);
 }
 
@@ -203,7 +219,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t
 #endif
 
     if(!status) {
-	if (nc5->pnetcdf_access_mode == NC_INDEPENDENT)
+        fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
+	if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP))
 	    status = ncmpi_begin_indep_data(nc->int_ncid);
     }
     return status;
@@ -611,7 +628,7 @@ NCP_get_vara(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break;
@@ -705,7 +722,7 @@ NCP_put_vara(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break;
@@ -801,7 +818,7 @@ NCP_get_vars(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break;
@@ -897,7 +914,7 @@ NCP_put_vars(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break;
@@ -995,7 +1012,7 @@ NCP_get_varm(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break;
@@ -1093,7 +1110,7 @@ NCP_put_varm(int ncid,
         if (status) return status;
     }
 
-    if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
 	switch(memtype) {
 	case NC_BYTE:
 	    status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break;
@@ -1198,19 +1215,43 @@ NCP_var_par_access(int ncid, int varid, int par_access)
     if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE)
 	return NC_EINVAL;
 
+#ifdef _DO_NOT_IGNORE_VARID_
+    if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */
+        return NC_EINVAL;
+#endif
+
     status = NC_check_id(ncid, &nc);
     if(status != NC_NOERR) return status;
 
     nc5 = NCP_DATA(nc);
     assert(nc5);
 
-    if(par_access == nc5->pnetcdf_access_mode)
-	return NC_NOERR;
-    nc5->pnetcdf_access_mode = par_access;
-    if (par_access == NC_INDEPENDENT)
-	return ncmpi_begin_indep_data(nc->int_ncid);
-    else
-	return ncmpi_end_indep_data(nc->int_ncid);
+    /* if currently in data mode */
+    if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) {
+        if (par_access == NC_INDEPENDENT) {
+            if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP))
+                return NC_NOERR;
+            else { /* currently in collective data mode */
+                fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
+                return ncmpi_begin_indep_data(nc->int_ncid);
+            }
+        }
+        else { /* want to enter collective data mode */
+            if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
+                fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
+                return ncmpi_end_indep_data(nc->int_ncid);
+            }
+            else
+                return NC_NOERR;
+        }
+    }
+    else { /* currently in define mode */
+        if (par_access == NC_INDEPENDENT)
+            fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
+        else
+            fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
+    }
+    return NC_NOERR;
 }
 
 #ifdef USE_NETCDF4

nc_test/tst_parallel2.c

@@ -135,8 +135,13 @@ main(int argc, char **argv)
        sleep(mpi_rank);
 #endif /* USE_MPE */
 
+#ifdef USE_PNETCDF
+/*    if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/
+    if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR;
+#else
 /*    if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/
     if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
+#endif
 
     if (!mpi_rank)
        start_time = MPI_Wtime();

nc_test/tst_pnetcdf.c

@@ -68,10 +68,8 @@ int main(int argc, char* argv[])
     }
     if (nc_enddef(ncid)) ERR;
 
-    for (i=0; i<NVARS; i++) {
-        /* Note NC_INDEPENDENT is the default */
-        if (nc_var_par_access(ncid, varid[i], NC_INDEPENDENT)) ERR;
-    }
+    /* Note NC_INDEPENDENT is the default */
+    if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR;
 
     /* write all variables */
     buf = (int*) malloc(NX * sizeof(int));

nc_test/tst_small.c

@@ -34,7 +34,7 @@
      int format; \
      nc_inq_format_extended(ncid,&format,NULL); \
      if (format == NC_FORMATX_PNETCDF) { \
-       if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;\
+       if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;\
      }\
    }
 #else
@@ -425,9 +425,7 @@ test_two_growing_with_att(const char *testfile)
       {int format;
       nc_inq_format_extended(ncid,&format,NULL);
       if (format == NC_FORMATX_PNETCDF) {
-          for (v = 0; v < NUM_VARS; v++) {
-              if (nc_var_par_access(ncid, varid[v], NC_COLLECTIVE)) ERR;
-          }
+          if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;
       }}
 #endif
       count[0] = 1;

nc_test/util.c

@@ -927,14 +927,8 @@ write_file(char *filename)
 	error("nc_enddef: %s", nc_strerror(err));
 
 #ifdef USE_PNETCDF
-    { int i,format;
-    nc_inq_format_extended(ncid, &format, NULL);
-    if (format == NC_FORMATX_PNETCDF) {
-        for (i = 0; i < numVars; i++) {
-            err = nc_var_par_access(ncid, i, NC_COLLECTIVE);
-	    IF (err) error("nc_var_par_access: %s", nc_strerror(err));
-        }
-    }}
+    err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE);
+    IF (err) error("nc_var_par_access: %s", nc_strerror(err));
 #endif
 
     put_vars(ncid);