1 new commit in MOAB: https://bitbucket.org/fathomteam/moab/commits/f48ec38dbf3e/ Changeset: f48ec38dbf3e Branch: master User: iulian07 Date: 2014-04-29 23:32:58 Summary: first pass at append option small test now for euler only maybe it should be tested for more formats assumes that if a dimension is already defined, there should be a variable with the same name (coordinate) Affected #: 5 files diff --git a/src/io/NCWriteHelper.cpp b/src/io/NCWriteHelper.cpp index 4f95ef8..923fe02 100644 --- a/src/io/NCWriteHelper.cpp +++ b/src/io/NCWriteHelper.cpp @@ -209,7 +209,7 @@ ErrorCode NCWriteHelper::collect_variable_data(std::vector<std::string>& var_nam return MB_SUCCESS; } -ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names) +ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names, bool append) { std::vector<std::string>& dimNames = _writeNC->dimNames; std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates; @@ -221,6 +221,9 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names) int tDim_in_dimNames = tDim; int levDim_in_dimNames = levDim; + // if append mode, make sure we are in define mode; a simple open will not allow creation of new variables + if (append) + NCFUNC(redef)(_fileId); // First initialize all coordinates, then fill VarData for actual variables (and dimensions) // Check that for used coordinates we have found the tags for (std::set<std::string>::iterator setIt = usedCoordinates.begin(); @@ -234,6 +237,40 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names) WriteNC::VarData& varCoordData = vit->second; varCoordData.varDims.resize(1); + // if not append, create it for sure + // if append, we might already have it, including the tag / variable with the same name + /* + * int ncmpi_inq_dimid(int ncid, const char *name, int *idp); + */ + if (append) + { + int dimId; + if (NCFUNC(inq_dimid)(_fileId, coordName.c_str(), &dimId) == NC_NOERR) + // if not found, create it later + { + varCoordData.varDims[0] = dimId; + // check that the coordinate is a variable too + // Skip dummy coordinate variables (e.g. ncol) + dbgOut.tprintf(2, " file already has coordName %s dim id is %d \n", coordName.c_str(), (int)varCoordData.varDims[0]); + if (dummyVarNames.find(coordName) != dummyVarNames.end()) + continue; + // inquire for a variable with the same name + + int varId; + if (NCFUNC(inq_varid)(_fileId, coordName.c_str(), &varId) != NC_NOERR) + ERRORR(MB_FAILURE, "we do not have a variable with the same name."); + // we should also check that this variable has one dimension, and it is dimId + varCoordData.varId = varId; + dbgOut.tprintf(2, " file already has coordinate %s and varId is %d \n", coordName.c_str(), varId); + // Update tDim and levDim to actual dimension id + if (coordName == dimNames[tDim_in_dimNames]) + tDim = varCoordData.varDims[0]; + else if (coordName == dimNames[levDim_in_dimNames]) + levDim = varCoordData.varDims[0]; + continue; // maybe more checks are needed here + } + + } /* int nc_def_dim (int ncid, const char *name, size_t len, int *dimidp); * example: status = nc_def_dim(fileId, "lat", 18L, &latid); */ @@ -297,9 +334,10 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names) } // Define the variable now: - if (NCFUNC(def_var)(_fileId, var_names[i].c_str(), variableData.varDataType, + int errCode = NCFUNC(def_var)(_fileId, var_names[i].c_str(), variableData.varDataType, (int)variableData.varDims.size(), &(variableData.varDims[0]), - &variableData.varId) != NC_NOERR) + &variableData.varId); + if ( errCode != NC_NOERR) ERRORR(MB_FAILURE, "Failed to create coordinate variable."); dbgOut.tprintf(2, " for variable %s variable id is %d \n", var_names[i].c_str(), variableData.varId); diff --git a/src/io/NCWriteHelper.hpp b/src/io/NCWriteHelper.hpp index b05fe12..ed2443a 100644 --- a/src/io/NCWriteHelper.hpp +++ b/src/io/NCWriteHelper.hpp @@ -34,7 +34,7 @@ public: //! Initialize file: this is where all defines are done //! The VarData dimension ids are filled up after define - ErrorCode init_file(std::vector<std::string>& var_names); + ErrorCode init_file(std::vector<std::string>& var_names, bool _append); protected: template <typename T> void jik_to_kji(size_t ni, size_t nj, size_t nk, T* dest, T* source) @@ -50,7 +50,7 @@ protected: //! Allow NCWriteHelper to directly access members of WriteNC WriteNC* _writeNC; - //! Cache some information from ReadNC + //! Cache some information from WriteNC int _fileId; const FileOptions& _opts; EntityHandle _fileSet; diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp index e665e26..98f4e3b 100644 --- a/src/io/WriteNC.cpp +++ b/src/io/WriteNC.cpp @@ -33,7 +33,7 @@ WriteNC::WriteNC(Interface* impl) : #ifdef USE_MPI myPcomm(NULL), #endif - noMesh(false), noVars(false), + noMesh(false), noVars(false), append(false), mGlobalIdTag(0), isParallel(false), myHelper(NULL) { @@ -84,23 +84,42 @@ ErrorCode WriteNC::write_file(const char* file_name, rval = process_conventional_tags(*file_set); ERRORR(rval, "Trouble processing conventional tags."); - // Create the file ; assume we will overwrite always, for the time being - dbgOut.tprintf(1, "creating file %s\n", file_name); + // Create or append the file + if (append) + dbgOut.tprintf(1, "opening file %s for appending \n", file_name); + else + dbgOut.tprintf(1, "creating file %s\n", file_name); fileName = file_name; int success; + if (append) + { #ifdef PNETCDF_FILE - int cmode = overwrite ? NC_CLOBBER : NC_NOCLOBBER; - if (isParallel) - success = NCFUNC(create)(myPcomm->proc_config().proc_comm(), file_name, cmode, MPI_INFO_NULL, &fileId); - else - success = NCFUNC(create)(MPI_COMM_SELF, file_name, cmode, MPI_INFO_NULL, &fileId); + int omode = NC_WRITE; + if (isParallel) + success = NCFUNC(open)(myPcomm->proc_config().proc_comm(), file_name, omode, MPI_INFO_NULL, &fileId); + else + success = NCFUNC(open)(MPI_COMM_SELF, file_name, omode, MPI_INFO_NULL, &fileId); #else - // This is a regular netcdf file - success = NCFUNC(create)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId); + // This is a regular netcdf file + success = NCFUNC(open)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId); #endif - ERRORS(success, "Failed to create file."); - + ERRORS(success, "Failed to open file for appending."); + } + else // case when the file is new, will be overwritten, most likely + { +#ifdef PNETCDF_FILE + int cmode = overwrite ? NC_CLOBBER : NC_NOCLOBBER; + if (isParallel) + success = NCFUNC(create)(myPcomm->proc_config().proc_comm(), file_name, cmode, MPI_INFO_NULL, &fileId); + else + success = NCFUNC(create)(MPI_COMM_SELF, file_name, cmode, MPI_INFO_NULL, &fileId); +#else + // This is a regular netcdf file + success = NCFUNC(create)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId); +#endif + ERRORS(success, "Failed to create file."); + } if (NULL != myHelper) delete myHelper; @@ -116,7 +135,7 @@ ErrorCode WriteNC::write_file(const char* file_name, rval = myHelper->collect_variable_data(var_names); ERRORR(rval, "Trouble collecting variable data."); - rval = myHelper->init_file(var_names); + rval = myHelper->init_file(var_names, append); ERRORR(rval, "Failed to initialize file."); rval = myHelper->write_values(var_names); @@ -148,6 +167,10 @@ ErrorCode WriteNC::parse_options(const FileOptions& opts, std::vector<std::strin if (MB_SUCCESS == rval) noMesh = true; + rval = opts.get_null_option("APPEND"); + if (MB_SUCCESS == rval) + append = true; + if (2 <= dbgOut.get_verbosity()) { if (!var_names.empty()) { std::cerr << "Variables requested: "; diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp index 1c5d78e..5827680 100644 --- a/src/io/WriteNC.hpp +++ b/src/io/WriteNC.hpp @@ -185,6 +185,7 @@ private: //! Write options bool noMesh; bool noVars; + bool append; //! Cached tags for writing. This will be important for ordering the data, in parallel Tag mGlobalIdTag; diff --git a/test/io/write_nc.cpp b/test/io/write_nc.cpp index 2d9bdae..f48e4c6 100644 --- a/test/io/write_nc.cpp +++ b/test/io/write_nc.cpp @@ -36,6 +36,7 @@ static const char example_mpas[] = "/io/mpasx1.642.t.2.nc"; // CAM-EUL void test_eul_read_write_T(); void test_eul_check_T(); +void test_eul_read_write_append(); // CAM-FV void test_fv_read_write_T(); @@ -68,12 +69,12 @@ int main(int argc, char* argv[]) result += RUN_TEST(test_eul_read_write_T); result += RUN_TEST(test_eul_check_T); + result += RUN_TEST(test_eul_read_write_append); result += RUN_TEST(test_fv_read_write_T); - result += RUN_TEST(test_fv_check_T); result += RUN_TEST(test_homme_read_write_T); result += RUN_TEST(test_homme_check_T); - //result += RUN_TEST(test_mpas_read_write_vars); - //result += RUN_TEST(test_mpas_check_vars); + // result += RUN_TEST(test_mpas_read_write_vars); + // result += RUN_TEST(test_mpas_check_vars); #ifdef USE_MPI fail = MPI_Finalize(); @@ -234,6 +235,61 @@ void test_eul_check_T() } } } +// We read and write variables T and U; U after we write T, so we append +// we will also write gw, just to test the file after writing +void test_eul_read_write_append() +{ + int procs = 1; +#ifdef USE_MPI + MPI_Comm_size(MPI_COMM_WORLD, &procs); +#endif + +// We will not test NC writer in parallel without pnetcdf support +#ifndef PNETCDF_FILE + if (procs > 1) + return; +#endif + + Core moab; + Interface& mb = moab; + + std::string read_opts; + get_eul_read_options(read_opts); + + EntityHandle set; + ErrorCode rval = mb.create_meshset(MESHSET_SET, set); + CHECK_ERR(rval); + + // Load non-set variable T, set variable gw, and the mesh + read_opts += ";DEBUG_IO=0;VARIABLE=T,U,gw"; + rval = mb.load_file(example_eul, &set, read_opts.c_str()); + CHECK_ERR(rval); + + // Write variables T and gw + std::string write_opts; + write_opts = std::string(";;VARIABLE=T,gw;DEBUG_IO=0"); +#ifdef USE_MPI + // Use parallel options + write_opts += std::string(";PARALLEL=WRITE_PART"); +#endif + if (procs > 1) + rval = mb.write_file("test_par_eul_TU.nc", 0, write_opts.c_str(), &set, 1); + else + rval = mb.write_file("test_eul_TU.nc", 0, write_opts.c_str(), &set, 1); + CHECK_ERR(rval); + // append to the file variable U + std::string write_opts2; + write_opts2 = std::string(";;VARIABLE=U;DEBUG_IO=0;APPEND"); +#ifdef USE_MPI + // Use parallel options + write_opts2 += std::string(";PARALLEL=WRITE_PART"); +#endif + if (procs > 1) + rval = mb.write_file("test_par_eul_TU.nc", 0, write_opts2.c_str(), &set, 1); + else + rval = mb.write_file("test_eul_TU.nc", 0, write_opts2.c_str(), &set, 1); + CHECK_ERR(rval); +} // We also write coordinate variables slat and slon to the output file, so that // it can be recognized by NC reader later in test_fv_check_T() Repository URL: https://bitbucket.org/fathomteam/moab/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.