2 new commits in MOAB: https://bitbucket.org/fathomteam/moab/commits/9bfb3fe9c697/ Changeset: 9bfb3fe9c697 Branch: None User: iulian07 Date: 2014-03-30 03:24:41 Summary: process attributes for each variable and global attributes these are saved when the original file is read with nomesh, novars option the attributes are concatenated, I saved them all as char do we really need them AttData is different compared to ReadNC why save the name again? well, not sure anybody needs them yet Affected #: 2 files diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp index a65548a..5af6fdf 100644 --- a/src/io/WriteNC.cpp +++ b/src/io/WriteNC.cpp @@ -13,6 +13,7 @@ #include <set> #include <iostream> +#include <sstream> #define ERRORR(rval, str) \ if (MB_SUCCESS != rval) { mWriteIface->report_error("%s", str); return rval; } @@ -365,6 +366,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) ERRORR(rval, " size of dimensions for variable"); dbgOut.tprintf(2, "var name: %s has %d dimensions \n", var_name.c_str(), sz); + variableDataStruct.varDims.resize(sz); //std::vector<const pcdim*> dims(sz, NULL); const void* ptr = NULL; rval = mbImpl->tag_get_by_ptr(dims_tag, &fileSet, 1, &ptr); @@ -376,123 +378,124 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) rval = mbImpl->tag_get_name(ptags[j], dim_name); ERRORR(rval, "name of tag for dimension"); dbgOut.tprintf(2, "var name: %s has %s as dimension \n", var_name.c_str(), dim_name.c_str() ); + std::vector<std::string>::iterator vit=std::find(dimNames.begin(), dimNames.end(), dim_name); + if (vit==dimNames.end()) + ERRORR(MB_FAILURE, "dimension not found\n"); + variableDataStruct.varDims[j]=(int)(vit-dimNames.begin()); // will be used for writing + // do we have a variable for each dimension? I mean, a tag? //dims[j] = &(get_dim(dim_name)); } - /*insert(var_name, - *(new fvar(var_name, moab::MB_TYPE_DOUBLE, dims, - locmap[varLoc[nthVar++]])));*/ + + // attributes for this variable + std::stringstream ssTagName; + ssTagName << "__" << var_name << "_ATTRIBS"; + tag_name = ssTagName.str(); + Tag varAttTag = 0; + rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_OPAQUE, varAttTag, MB_TAG_SPARSE | MB_TAG_VARLEN); + ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS tag."); + std::string varAttVal; + std::vector<int> varAttLen; + const void* varAttPtr = 0; + int varAttSz = 0; + rval = mbImpl->tag_get_by_ptr(varAttTag, &fileSet, 1, &varAttPtr, &varAttSz); + ERRORR(rval, "Trouble setting data for __<var_name>_ATTRIBS tag."); + if (MB_SUCCESS == rval) + dbgOut.tprintf(2, "Tag retrieved for variable %s\n", tag_name.c_str()); + + ssTagName << "_LEN"; + tag_name = ssTagName.str(); + Tag varAttLenTag = 0; + rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_INTEGER, varAttLenTag, MB_TAG_ANY); + ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS_LEN tag."); + int varAttLenSz=0; + rval = mbImpl->tag_get_length(varAttLenTag, varAttLenSz); + ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS_LEN length."); + varAttLen.resize(varAttLenSz); + + rval = mbImpl->tag_get_data(varAttLenTag, &fileSet, 1, &varAttLen[0]); + ERRORR(rval, "Trouble getting data for __<var_name>_ATTRIBS_LEN tag."); + + rval = process_concatenated_attribute(varAttPtr, varAttSz, varAttLen, variableDataStruct.varAtts); + ERRORR(rval, " trouble processing global attributes "); + + if (MB_SUCCESS == rval) + dbgOut.tprintf(2, "Tag metadata for variable %s\n", tag_name.c_str()); + // end attribute start = i + 1; idxVar++; } } // attributes -#if 0 - std::vector<std::string> nameVec(num_vars+1); - nameVec[0] = "GLOBAL"; - std::map<std::string, const fvar*>::iterator it = m_vars.begin(); - for (std::size_t i=1; it != m_vars.end(); ++it, ++i) - nameVec[i] = it->first; - - for (std::size_t vec_counter = 0; vec_counter != nameVec.size(); ++vec_counter) - { - // read __<var_name>_ATTRIBS tag - moab::Tag tag = 0; - std::string tag_name = "__"+nameVec[vec_counter]+"_ATTRIBS"; - const void * data = NULL; - int sz = 0; - moab::ErrorCode rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY); - if (rval != moab::MB_SUCCESS) - throw pargal_except("Error: " + m_mb.get_error_string(rval), - __FILE__, __LINE__, __PRETTY_FUNCTION__); - - rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz); - const char * p = static_cast<const char *>(data); - std::string att_val(&p[0], sz); - if (vec_counter == 0) nameVec[0]="MOAB_GLOBAL"; - const std::string& var_name = nameVec[vec_counter]; - - // read __<var_name>_ATTRIBS_LEN tag - moab::Tag attLenTag = 0; - tag_name = tag_name + "_LEN"; - const void * len_data = NULL; - int len_sz = 0; - rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_INTEGER, attLenTag, moab::MB_TAG_ANY); - rval = m_mb.tag_get_by_ptr(attLenTag, &m_file_set, 1, &len_data, &len_sz); - const int * len_p = static_cast<const int *>(len_data); - std::vector<int> attLen(len_sz); - std::copy(len_p, len_p+len_sz, attLen.begin()); - - // create attribute - insert(var_name, *(new pcatt(var_name, att_val, attLen))); -#endif + tag_name = "__GLOBAL_ATTRIBS"; + Tag globalAttTag = 0; + rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_OPAQUE, globalAttTag, MB_TAG_SPARSE | MB_TAG_VARLEN); + ERRORR(rval, "Trouble getting __GLOBAL_ATTRIBS tag."); + std::string gattVal; + std::vector<int> gattLen; + + const void* gattptr; + int globalAttSz=0; + rval = mbImpl->tag_get_by_ptr(globalAttTag, &fileSet, 1, &gattptr, &globalAttSz); + ERRORR(rval, "Trouble getting data for __GLOBAL_ATTRIBS tag."); + + if (MB_SUCCESS == rval) + dbgOut.tprintf(2, "Tag value retrieved for %s size %d\n", tag_name.c_str(), globalAttSz); + + // <__GLOBAL_ATTRIBS_LEN> + tag_name = "__GLOBAL_ATTRIBS_LEN"; + Tag globalAttLenTag = 0; + + rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_INTEGER, globalAttLenTag, MB_TAG_ANY); + ERRORR(rval, "Trouble getting __GLOBAL_ATTRIBS_LEN tag."); + int sizeGAtt =0; + rval = mbImpl->tag_get_length(globalAttLenTag, sizeGAtt); + ERRORR(rval, "Trouble getting length of __GLOBAL_ATTRIBS_LEN tag "); + gattLen.resize(sizeGAtt); + rval = mbImpl->tag_get_data(globalAttLenTag, &fileSet, 1, &gattLen[0]); + ERRORR(rval, "Trouble setting data for __GLOBAL_ATTRIBS_LEN tag."); + if (MB_SUCCESS == rval) + dbgOut.tprintf(2, "Tag retrieved for variable %s\n", tag_name.c_str()); + + rval = process_concatenated_attribute(gattptr, globalAttSz, gattLen, globalAtts); + ERRORR(rval, " trouble processing global attributes "); + return MB_SUCCESS; } -#if 0 +ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen, + std::map<std::string, AttData> & attributes) +{ - void - fileinfo::init_atts() - { - std::vector<std::string> nameVec(m_vars.size()+1); - nameVec[0] = "GLOBAL"; - std::map<std::string, const fvar*>::iterator it = m_vars.begin(); - for (std::size_t i=1; it != m_vars.end(); ++it, ++i) - nameVec[i] = it->first; - - for (std::size_t vec_counter = 0; vec_counter != nameVec.size(); ++vec_counter) - { - // read __<var_name>_ATTRIBS tag - moab::Tag tag = 0; - std::string tag_name = "__"+nameVec[vec_counter]+"_ATTRIBS"; - const void * data = NULL; - int sz = 0; - moab::ErrorCode rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY); - if (rval != moab::MB_SUCCESS) - throw pargal_except("Error: " + m_mb.get_error_string(rval), - __FILE__, __LINE__, __PRETTY_FUNCTION__); - - rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz); - const char * p = static_cast<const char *>(data); - std::string att_val(&p[0], sz); - if (vec_counter == 0) nameVec[0]="MOAB_GLOBAL"; - const std::string& var_name = nameVec[vec_counter]; - - // read __<var_name>_ATTRIBS_LEN tag - moab::Tag attLenTag = 0; - tag_name = tag_name + "_LEN"; - const void * len_data = NULL; - int len_sz = 0; - rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_INTEGER, attLenTag, moab::MB_TAG_ANY); - rval = m_mb.tag_get_by_ptr(attLenTag, &m_file_set, 1, &len_data, &len_sz); - const int * len_p = static_cast<const int *>(len_data); - std::vector<int> attLen(len_sz); - std::copy(len_p, len_p+len_sz, attLen.begin()); - - // create attribute - insert(var_name, *(new pcatt(var_name, att_val, attLen))); - } - } + std::size_t start = 0; + std::size_t att_counter = 0; + std::string concatString( (char*)gattptr, (char*)gattptr+globalAttSz); - void - fileinfo::init_grid_type() + for (std::size_t i = 0; i != (size_t)globalAttSz; ++i) + { + if (concatString[i] == '\0') { - moab::Tag tag = 0; - std::string tag_name = "__MESH_TYPE"; - const void * data = NULL; - int sz = 0; - moab::ErrorCode rval; - rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY); - rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz); - if (rval != moab::MB_SUCCESS) - throw pargal_except("Error: " + m_mb.get_error_string(rval), - __FILE__, __LINE__, __PRETTY_FUNCTION__); - const char * p = static_cast<const char *>(data); - m_grid_type = std::string(&p[0], sz); + std::string att_name(&concatString[start], i - start); + start = i + 1; + while (concatString[i] != ';') + ++i; + std::string data_type(&concatString[start], i - start); + ++i; + start = i; + i = gattLen[att_counter]; + if (concatString[i] != ';') + ERRORR(MB_FAILURE, "Error parsing attributes "); + + std::string data_val(&concatString[start], i - start); + start = i + 1; + AttData attrib = attributes[att_name]; + attrib.attValue = data_val; + ++att_counter; + dbgOut.tprintf(2, " Process attribute %s with value %s \n",att_name.c_str(), data_val.c_str() ); } + } return MB_SUCCESS; } -#endif } // end moab namespace diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp index c3b893a..6b75d59 100644 --- a/src/io/WriteNC.hpp +++ b/src/io/WriteNC.hpp @@ -110,7 +110,7 @@ private: NCDF_SIZE attLen; int attVarId; nc_type attDataType; - std::string attName; + std::string attValue; }; class VarData @@ -152,6 +152,9 @@ private: * ErrorCode NCHelper::create_conventional_tags */ ErrorCode process_conventional_tags(EntityHandle fileSet); + + ErrorCode process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen, + std::map<std::string, AttData> & globalAtts); // interface instance Interface *mbImpl; WriteUtilIface* mWriteIface; https://bitbucket.org/fathomteam/moab/commits/e64001b99718/ Changeset: e64001b99718 Branch: ncwriter User: iulian07 Date: 2014-03-30 17:23:36 Summary: start collecting data will have to populate VarData with appropriate info; start gathering tag info for nonset variables "set" variables are coordinate variables (dimensions), usually time is integer (time steps, or time values?) Affected #: 2 files diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp index 5af6fdf..05fcc3f 100644 --- a/src/io/WriteNC.cpp +++ b/src/io/WriteNC.cpp @@ -122,6 +122,9 @@ ErrorCode WriteNC::write_file(const char *file_name, const size_t count[], const double *dp); */ + // + rval = collect_variable_data(var_names, tstep_nums, tstep_vals, *file_set); + ERRORR(rval, "Trouble collecting data."); return MB_SUCCESS; } @@ -348,7 +351,11 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) dbgOut.tprintf(2, "var name: %s index %d \n", var_name.c_str(), idxVar); // process var name: // this will create/initiate map; we will populate variableDataStruct wit info about dims, tags, etc - VarData variableDataStruct = varInfo[var_name]; + // reference & is important; otherwise variableDataStruct will go out of scope, and deleted :( + VarData & variableDataStruct = varInfo[var_name]; + + dbgOut.tprintf(2, "at var name %s varInfo size %d \n", var_name.c_str(), varInfo.size() ); + sz = 0; Tag dims_tag = 0; std::string dim_names = "__" + var_name + "_DIMS"; @@ -371,6 +378,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) const void* ptr = NULL; rval = mbImpl->tag_get_by_ptr(dims_tag, &fileSet, 1, &ptr); // + const Tag * ptags = static_cast<const moab::Tag*>(ptr); for (std::size_t j = 0; j != static_cast<std::size_t>(sz); ++j) { @@ -381,7 +389,8 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) std::vector<std::string>::iterator vit=std::find(dimNames.begin(), dimNames.end(), dim_name); if (vit==dimNames.end()) ERRORR(MB_FAILURE, "dimension not found\n"); - variableDataStruct.varDims[j]=(int)(vit-dimNames.begin()); // will be used for writing + variableDataStruct.varDims[j]= (int)(vit-dimNames.begin()) ; // will be used for writing + // do we have a variable for each dimension? I mean, a tag? //dims[j] = &(get_dim(dim_name)); } @@ -462,7 +471,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet) return MB_SUCCESS; } - +// reverse process from create_attrib_string ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen, std::map<std::string, AttData> & attributes) { @@ -498,4 +507,106 @@ ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int glob return MB_SUCCESS; } +ErrorCode WriteNC::collect_variable_data( std::vector<std::string>& var_names, std::vector<int>& tstep_nums, + std::vector<double>& tstep_vals, EntityHandle fileSet) +{ + // in general, in netcdf, variables that have the same name as their only dimension are called + // coordinate variables + // for the time being, check if all dimensions for variables are coordinate variables + ErrorCode rval; + + usedCoordinates.clear(); + + for (size_t i=0; i<var_names.size(); i++) + { + std::string varname=var_names[i]; + std::map<std::string, VarData>::iterator vit = varInfo.find(varname); + if (vit==varInfo.end()) + ERRORR(MB_FAILURE, "can't find one variable"); + + size_t sizeVar = 1;// get multiplied by dim lengths + VarData & currentVarData = vit->second; + dbgOut.tprintf(2, " for variable %s varDims.size %d \n", varname.c_str(), (int)currentVarData.varDims.size() ); + for (size_t j =0; j<currentVarData.varDims.size(); j++) + { + std::string dimName= dimNames[ currentVarData.varDims[j] ]; + vit = varInfo.find(dimName); + if (vit==varInfo.end()) + ERRORR(MB_FAILURE, "can't find one coordinate variable"); + + if( (dimName == "time" || dimName == "Time" || dimName == "t") && + currentVarData.varDims.size()>1 ) // so it is not time itself + currentVarData.has_tsteps=true; + + // probably will have to look at tstep_vals to match them + sizeVar *= dimLens[j]; + usedCoordinates.insert(dimName); // collect those used, we will need to write them to the file + dbgOut.tprintf(2, " for variable %s need dimension %s with length %d\n", varname.c_str(), dimName.c_str(), dimLens[j] ); + } + + currentVarData.sz=sizeVar; + + if (currentVarData.has_tsteps) + { + + int index=0; + while(1) + { + Tag indexedTag; + std::stringstream ssTagNameWithIndex; + ssTagNameWithIndex << varname << index; + rval = mbImpl->tag_get_handle(ssTagNameWithIndex.str().c_str(), indexedTag); + if (rval!=MB_SUCCESS) + break; + dbgOut.tprintf(2, " found indexed tag %d with name %s\n", index, ssTagNameWithIndex.str().c_str()); + currentVarData.varTags.push_back(indexedTag); + index++; // we should get out of the loop at some point + // we will have to collect data for these tags; maybe even allocate memory again + } + } + else + { + // get the tag with varname + Tag coordtag=0; + rval = mbImpl->tag_get_handle(varname.c_str(), coordtag); + ERRORR(rval, "can't find one tag"); + currentVarData.varTags.push_back(coordtag); // really, only one for these + const void * data; + int sizeCoordinate; + rval = mbImpl->tag_get_by_ptr(coordtag, &fileSet, 1, &data, &sizeCoordinate); + ERRORR(rval, "can't get coordinate values"); + assert(currentVarData.memoryHogs.size()==0);// nothing so far + currentVarData.memoryHogs.push_back((void*)data); + } + + } + + // check that for used coordinates we have found the tags + for (std::set<std::string>::iterator setIt = usedCoordinates.begin(); setIt!=usedCoordinates.end(); setIt++) + { + std::string coordName=*setIt; // a deep copy ; is it needed? + + std::map<std::string, VarData>::iterator vit = varInfo.find(coordName); + if (vit==varInfo.end()) + ERRORR(MB_FAILURE, "can't find one coordinate variable"); + + VarData & varCoordData = vit->second; + Tag coordtag=0; + rval = mbImpl->tag_get_handle(coordName.c_str(), coordtag); + ERRORR(rval, "can't find one tag"); + varCoordData.varTags.push_back(coordtag); // really, only one for these + + const void * data; + int sizeCoordinate; + rval = mbImpl->tag_get_by_ptr(coordtag, &fileSet, 1, &data, &sizeCoordinate); + ERRORR(rval, "can't get coordinate values"); + dbgOut.tprintf(2, " found coordinate tag with name %s and length %d\n", coordName.c_str(), + sizeCoordinate); + + assert(varCoordData.memoryHogs.size()==0);// nothing so far + varCoordData.memoryHogs.push_back((void*)data); + } + return MB_SUCCESS; +} + } // end moab namespace diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp index 6b75d59..df89af1 100644 --- a/src/io/WriteNC.hpp +++ b/src/io/WriteNC.hpp @@ -139,6 +139,9 @@ private: //! Dimension lengths std::vector<int> dimLens; + // will collect used dimensions (ccordinate variables) + std::set<std::string> usedCoordinates; + //! Global attribs std::map<std::string, AttData> globalAtts; @@ -155,6 +158,11 @@ private: ErrorCode process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen, std::map<std::string, AttData> & globalAtts); + + // will collect data; it should be only on gather processor, but for the time being, collect + // for everybody + ErrorCode collect_variable_data( std::vector<std::string>& var_names, std::vector<int>& tstep_nums, + std::vector<double>& tstep_vals, EntityHandle fileSet); // interface instance Interface *mbImpl; WriteUtilIface* mWriteIface; Repository URL: https://bitbucket.org/fathomteam/moab/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.