14#include "arcane/impl/internal/VariableMng.h"
16#include "arcane/utils/FatalErrorException.h"
17#include "arcane/utils/ParallelFatalErrorException.h"
18#include "arcane/utils/IObservable.h"
19#include "arcane/utils/ScopedPtr.h"
20#include "arcane/utils/StringBuilder.h"
21#include "arcane/utils/MD5HashAlgorithm.h"
22#include "arcane/utils/SHA1HashAlgorithm.h"
23#include "arcane/utils/OStringStream.h"
24#include "arcane/utils/PlatformUtils.h"
25#include "arcane/utils/JSONReader.h"
26#include "arcane/utils/ValueConvert.h"
28#include "arcane/core/IXmlDocumentHolder.h"
29#include "arcane/core/XmlNode.h"
30#include "arcane/core/VariableMetaData.h"
31#include "arcane/core/IData.h"
32#include "arcane/core/IMesh.h"
33#include "arcane/core/Properties.h"
34#include "arcane/core/Timer.h"
35#include "arcane/core/VarRefEnumerator.h"
36#include "arcane/core/CheckpointInfo.h"
37#include "arcane/core/Directory.h"
38#include "arcane/core/ISerializedData.h"
39#include "arcane/core/VariableBuildInfo.h"
40#include "arcane/core/XmlNodeList.h"
41#include "arcane/core/IMeshMng.h"
42#include "arcane/core/IMeshFactoryMng.h"
43#include "arcane/core/MeshBuildInfo.h"
44#include "arcane/core/internal/IDataInternal.h"
46#include "arcane/core/ICheckpointReader.h"
47#include "arcane/core/IDataReader.h"
48#include "arcane/core/IDataReader2.h"
50#include "arcane/core/ISubDomain.h"
51#include "arcane/core/IParallelMng.h"
53#include "arcane/core/internal/IParallelMngInternal.h"
54#include "arcane/core/internal/IVariableInternal.h"
74 explicit VariableDataTypeInfo(
const String& full_type)
77 std::vector<String> split_strs;
78 full_type.
split(split_strs,
'.');
79 size_t nb_split = split_strs.size();
81 if (split_strs[4] !=
"Partial")
82 ARCANE_FATAL(
"Invalid value for partial full_type '{0}'", full_type);
87 ARCANE_FATAL(
"Invalid value for full_type '{0}'", full_type);
88 m_data_type_name = split_strs[0];
89 m_item_kind_name = split_strs[1];
90 m_dimension = split_strs[2];
91 m_multi_tag = split_strs[3];
96 const String& dataTypeName()
const {
return m_data_type_name; }
97 const String& itemKindName()
const {
return m_item_kind_name; }
98 const String& dimension()
const {
return m_dimension; }
99 const String& multiTag()
const {
return m_multi_tag; }
100 bool isPartial()
const {
return m_is_partial; }
126 virtual void endRead() = 0;
144 return m_reader->beginRead(vars);
149 m_reader->read(var, data);
151 void endRead()
override
153 return m_reader->endRead();
184 void endRead()
override
186 return m_reader->endRead();
197class VariableMetaDataList
201 typedef std::map<String, Ref<VariableMetaData>> VMDMap;
205 VariableMetaDataList() =
default;
206 VariableMetaDataList(
const VariableMetaDataList& rhs) =
delete;
207 VariableMetaDataList& operator=(
const VariableMetaDataList& rhs) =
delete;
208 ~VariableMetaDataList()
224 m_vmd_map.insert(std::make_pair(vmd->fullName(), vmd));
233 auto x = m_vmd_map.find(full_name);
234 if (x != m_vmd_map.end())
235 return x->second.get();
238 VMDMap::const_iterator begin()
const {
return m_vmd_map.begin(); }
239 VMDMap::const_iterator end()
const {
return m_vmd_map.end(); }
240 void setHashAlgorithmName(
const String& v) { m_hash_algorithm = v; }
241 const String& hashAlgorithmName()
const {
return m_hash_algorithm; }
245 std::map<String, Ref<VariableMetaData>> m_vmd_map;
257class VariableReaderMng
267 , m_meta_data(meta_data)
279 explicit VariableReaderMng(
ITraceMng* tm)
287 const VariableList& variablesToRead() {
return m_vars_to_read; }
291 VariableMetaDataList m_vmd_list;
292 VariableList m_vars_to_read;
293 UniqueArray<VarReadInfo> m_var_read_info_list;
297 void _buildVariablesToRead(IVariableMng* vm);
303void VariableReaderMng::
306 m_vars_to_read.clear();
307 m_var_read_info_list.clear();
308 for (
const auto& x : m_vmd_list) {
309 const String& full_name = x.first;
310 IVariable* var = vm->findVariableFullyQualified(full_name);
313 m_vars_to_read.add(var);
314 m_var_read_info_list.add(
VarReadInfo(var, var->data(), x.second.get()));
321void VariableReaderMng::
324 _buildVariablesToRead(vm);
325 reader->beginRead(m_vars_to_read);
326 for (
const auto& ivar : m_var_read_info_list) {
328 IVariable* var = ivar.m_variable;
329 IData* data = ivar.m_data;
330 VariableMetaData* vmd = ivar.m_meta_data;
331 String exception_message;
332 bool has_error =
false;
334 reader->read(vmd, var, data);
336 var->notifyEndRead();
338 catch (
const Exception& ex) {
341 exception_message = ostr.str();
344 catch (
const std::exception& ex) {
345 exception_message = ex.what();
350 String var_full_name = vmd->fullName();
351 ostr() <<
"Variable = " << var_full_name;
353 for (VarRefEnumerator ivar(var); ivar.hasNext(); ++ivar) {
354 VariableRef* ref = *ivar;
355 String s = ref->assignmentStackTrace();
357 ostr() <<
"Stack assignement: " << s;
361 ARCANE_FATAL(
"Can not read variable variable={0} exception={1} infos={2}",
362 var_full_name, exception_message, ostr.str());
378, m_is_use_json_metadata(true)
381 m_is_use_json_metadata = (v.value() != 0);
390void VariableIOReaderMng::
391readCheckpoint(ICheckpointReader* service)
393 Trace::Setter mci(
traceMng(), _msgClassName());
398 service->notifyBeginRead();
400 IDataReader* data_reader = service->dataReader();
403 String meta_data = data_reader->metaData();
405 if (meta_data.null())
408 OldDataReaderWrapper wrapper(data_reader);
410 info(6) <<
"METADATA (ICheckpointReader): FromCheckpoint: " << meta_data;
412 VariableReaderMng var_read_mng(
traceMng());
413 VariableMetaDataList& vmd_list = var_read_mng.variableMetaDataList();
415 _readMetaData(vmd_list, meta_data.bytes());
416 _createVariablesFromMetaData(vmd_list);
417 _readVariablesData(var_read_mng, &wrapper);
420 service->notifyEndRead();
421 _checkHashFunction(vmd_list);
427void VariableIOReaderMng::
428readCheckpoint(
const CheckpointReadInfo& infos)
430 Trace::Setter mci(
traceMng(), _msgClassName());
432 ICheckpointReader2* service = infos.reader();
435 IParallelMng* pm = infos.parallelMng();
438 service->notifyBeginRead(infos);
440 IDataReader2* data_reader = service->dataReader();
443 UniqueArray<Byte> meta_data_bytes;
444 data_reader->fillMetaData(meta_data_bytes);
445 if (meta_data_bytes.empty())
448 DataReaderWrapper wrapper(data_reader);
450 info(6) <<
"METADATA (ICheckpointReader2): FromCheckpoint: " << String(meta_data_bytes);
452 VariableReaderMng var_read_mng(
traceMng());
453 VariableMetaDataList& vmd_list = var_read_mng.variableMetaDataList();
455 _readMetaData(vmd_list, meta_data_bytes);
456 _createVariablesFromMetaData(vmd_list);
457 _readVariablesData(var_read_mng, &wrapper);
460 service->notifyEndRead();
461 _checkHashFunction(vmd_list);
467void VariableIOReaderMng::
468readVariables(IDataReader* reader, IVariableFilter* filter)
470 Trace::Setter mci(
traceMng(), _msgClassName());
471 VariableReaderMng var_read_mng(
traceMng());
472 _buildFilteredVariableList(var_read_mng, filter);
473 OldDataReaderWrapper wrapper(reader);
474 _readVariablesData(var_read_mng, &wrapper);
499 XmlNode root_node = doc->documentNode().documentElement();
507 if (!json_node.
null()) {
509 info(6) <<
"READER_JSON=" << json_meta_data;
516 String hash_algo_name = json_meta_data_object.
child(
"hash-algorithm-name").
value();
517 vmd_list.setHashAlgorithmName(hash_algo_name);
519 if (m_is_use_json_metadata) {
523 ARCANE_FATAL(
"Bad version for JSON Meta Data (v={0}). Only version '1' is supported", v);
524 json_variables = json_meta_data_object.
expectedChild(
"variables");
529 _readVariablesMetaData(vmd_list, json_variables, variables_node);
531 _readMeshesMetaData(json_meshes, meshes_node);
556 String hash_service_name = vmd_list.hashAlgorithmName();
557 if (!hash_service_name.
empty()) {
558 if (hash_service_name ==
"MD5")
559 hash_algo = &md5_hash_algorithm;
560 else if (hash_service_name ==
"SHA1") {
561 hash_algo = &sha1_hash_algorithm;
565 ARCANE_FATAL(
"Not supported hash algorithm '{0}'. Valid values are 'SHA1' or 'MD5'");
570 Directory listing_dir = m_variable_mng->subDomain()->listingDirectory();
571 for (
const auto& i : vmd_list) {
574 String reference_hash = (hash_version > 0) ? vmd->
hash2() : vmd->
hash();
579 if (reference_hash.
null())
581 const String& full_name = i.first;
582 IVariable* var = m_variable_mng->findVariableFullyQualified(full_name);
589 bool do_compare =
true;
590 if (hash_version > 0) {
592 hash_context->
reset();
596 hash_context->computeHashValue(hash_value);
599 if (hash_version != hash_info.version())
606 if (do_compare && (hash_str != reference_hash)) {
608 error() <<
"Hash values are different. Corrumpted values."
610 <<
" ref=" << reference_hash
611 <<
" current=" << hash_str;
614 String fname = listing_dir.
file(String::format(
"dump-{0}-sid_{1}", var->
fullName(), sid));
615 std::ofstream ofile(fname.
localstr());
616 ofile.write(
reinterpret_cast<const char*
>(buf.
data()), buf.
size());
620 if (total_nb_error != 0) {
630void VariableIOReaderMng::
635 for (
const auto& xvmd : vmd_list) {
636 auto& vmd = *(xvmd.second.get());
637 const String& full_name = vmd.fullName();
638 IVariable* var = m_variable_mng->findVariableFullyQualified(full_name);
641 const String& base_name = vmd.baseName();
642 Integer property = vmd.property();
643 const String& mesh_name = vmd.meshName();
644 const String& group_name = vmd.itemGroupName();
645 const String& family_name = vmd.itemFamilyName();
647 if (!mesh_name.
null()) {
649 vbi =
VariableBuildInfo(sd, base_name, mesh_name, family_name, group_name, property);
653 info(5) <<
"Create variable TYPE=" << full_name;
654 VariableRef* variable_ref = m_variable_mng->_createVariableFromType(vmd.fullType(), vbi);
659 if (!mesh_name.
null()) {
666 variable_ref->variable()->_internalApi()->changeAllocator(MemoryAllocationOptions(pm->_internalApi()->machineShMemWinMemoryAllocator()));
674void VariableIOReaderMng::
675_readVariablesMetaData(VariableMetaDataList& vmd_list, JSONValue variables_json,
676 const XmlNode& variables_node)
678 String ustr_base_name(
"base-name");
679 String ustr_family_name(
"item-family-name");
680 String ustr_group_name(
"item-group-name");
681 String ustr_mesh_name(
"mesh-name");
682 String ustr_full_type(
"full-type");
683 String ustr_data_type(
"data-type");
684 String ustr_hash(
"hash");
685 String ustr_hash2(
"hash2");
686 String ustr_hash_version(
"hash-version");
687 String ustr_property(
"property");
688 String ustr_multitag(
"multi-tag");
691 struct VariableReadInfo
700 Int32 hash_version = 0;
705 UniqueArray<VariableReadInfo> variables_info;
709 if (!variables_json.null()) {
712 JSONValueList vars = variables_json.valueAsArray();
713 for (
const JSONValue& var : vars) {
715 r.full_type = var.expectedChild(ustr_full_type).value();
716 r.base_name = var.expectedChild(ustr_base_name).value();
717 r.data_type = var.child(ustr_data_type).value();
718 r.mesh_name = var.child(ustr_mesh_name).value();
719 r.family_name = var.child(ustr_family_name).value();
720 r.group_name = var.child(ustr_group_name).value();
721 r.hash_value = var.child(ustr_hash).value();
722 r.hash2_value = var.child(ustr_hash2).value();
723 r.hash_version = var.child(ustr_hash_version).valueAsInt32();
724 r.multi_tag = var.child(ustr_multitag).value();
725 r.property = var.child(ustr_property).valueAsInt32();
726 variables_info.add(r);
731 XmlNodeList vars = variables_node.children(
"variable");
732 for (
const auto& var : vars) {
734 r.full_type = var.attrValue(ustr_full_type);
735 r.data_type = var.attrValue(ustr_data_type);
736 r.base_name = var.attrValue(ustr_base_name);
737 r.mesh_name = var.attrValue(ustr_mesh_name);
738 r.group_name = var.attrValue(ustr_group_name);
739 r.family_name = var.attrValue(ustr_family_name);
740 r.hash_value = var.attrValue(ustr_hash);
741 r.multi_tag = var.attrValue(ustr_multitag);
742 r.property = var.attr(ustr_property).valueAsInteger();
743 variables_info.add(r);
747 for (
const VariableReadInfo& r : variables_info) {
748 String full_type = r.full_type;
749 VariableDataTypeInfo vdti(full_type);
752 if (vdti.dataTypeName() != r.data_type)
753 ARCANE_FATAL(
"Incoherent value for 'data-type' name v='{0}' expected='{1}'", r.data_type, vdti.dataTypeName());
755 String family_name = r.family_name;
758 String group_name = r.group_name;
759 bool is_partial = vdti.isPartial();
764 group_name =
"All" + family_name +
"s";
766 auto vmd = vmd_list.add(r.base_name, r.mesh_name, r.family_name, group_name, is_partial);
768 vmd->setFullType(full_type);
769 vmd->setHash(r.hash_value);
770 vmd->setHash2(r.hash2_value);
771 vmd->setHashVersion(r.hash_version);
772 vmd->setMultiTag(r.multi_tag);
773 vmd->setProperty(r.property);
775 info(5) <<
"CHECK VAR: "
776 <<
" base-name=" << vmd->baseName()
777 <<
" mesh-name=" << vmd->meshName()
778 <<
" family-name=" << vmd->itemFamilyName()
779 <<
" full-type=" << vmd->fullType()
780 <<
" name=" << vmd->fullName()
781 <<
" multitag=" << vmd->multiTag()
782 <<
" property=" << vmd->property()
783 <<
" hash=" << vmd->hash();
790void VariableIOReaderMng::
791_readMeshesMetaData(JSONValue meshes_json,
const XmlNode& meshes_node)
793 ISubDomain* sd = m_variable_mng->subDomain();
794 IMeshMng* mesh_mng = sd->
meshMng();
803 UniqueArray<MeshInfo> meshes_info;
807 if (!meshes_json.null()) {
809 JSONValueList vars = meshes_json.valueAsArray();
810 for (
const JSONValue& var : vars) {
811 String mesh_name = var.expectedChild(
"name").value();
812 String mesh_factory_name = var.child(
"factory-name").value();
813 bool is_sequential =
false;
814 JSONValue v = var.child(
"sequential");
816 is_sequential = v.valueAsBool();
817 meshes_info.add({ mesh_name, mesh_factory_name, is_sequential });
821 XmlNodeList meshes = meshes_node.children(
"mesh");
822 for (XmlNode var : meshes) {
823 String mesh_name = var.attrValue(
"name");
824 String mesh_factory_name = var.attrValue(
"factory-name");
825 bool is_sequential = var.attr(
"sequential",
false).valueAsBoolean();
826 meshes_info.add({ mesh_name, mesh_factory_name, is_sequential });
830 for (
const MeshInfo& mesh_info : meshes_info) {
831 String mesh_name = mesh_info.name;
832 String mesh_factory_name = mesh_info.factory_name;
833 MeshHandle* mesh_handle = mesh_mng->findMeshHandle(mesh_name,
false);
834 IMesh* mesh = (mesh_handle) ? mesh_handle->mesh() :
nullptr;
837 bool is_sequential = mesh_info.is_sequential;
838 info() <<
"Creating from checkpoint mesh='" << mesh_name
839 <<
"' sequential?=" << is_sequential
840 <<
" factory=" << mesh_factory_name;
843 if (mesh_factory_name.null())
847 MeshBuildInfo mbi(mesh_name);
848 mbi.addFactoryName(mesh_factory_name);
849 IParallelMng* mesh_pm = m_variable_mng->m_parallel_mng;
851 mesh_pm = mesh_pm->sequentialParallelMng();
853 mesh_factory_mng->createMesh(mbi);
861void VariableIOReaderMng::
862_buildFilteredVariableList(VariableReaderMng& var_read_mng, IVariableFilter* filter)
864 VariableMetaDataList& vmd_list = var_read_mng.variableMetaDataList();
865 for (
const auto& i : m_variable_mng->m_full_name_variable_map) {
866 IVariable* var = i.second;
867 bool apply_me =
true;
869 apply_me = filter->applyFilter(*var);
870 info(5) <<
"Read variable name=" << var->fullName() <<
" filter=" << apply_me;
872 Ref<VariableMetaData> vmd = var->createMetaDataRef();
881void VariableIOReaderMng::
882_readVariablesData(VariableReaderMng& var_read_mng, IDataReaderWrapper* reader)
884 var_read_mng.readVariablesData(m_variable_mng, reader);
885 _finalizeReadVariables(var_read_mng.variablesToRead());
891void VariableIOReaderMng::
892_finalizeReadVariables(
const VariableList& vars_to_read)
894 ARCANE_UNUSED(vars_to_read);
896 info(4) <<
"VariableMng: _finalizeReadVariables()";
905 for (
const auto& i : m_variable_mng->m_full_name_variable_map)
906 i.second->syncReferences();
909 m_variable_mng->m_read_observable->notifyAllObservers();
#define ARCANE_CHECK_POINTER(ptr)
Macro retournant le pointeur ptr s'il est non nul ou lancant une exception s'il est nul.
#define ARCANE_CHECK_POINTER2(ptr, text)
Macro retournant le pointeur ptr s'il est non nul ou lancant une exception s'il est nul.
#define ARCANE_FATAL(...)
Macro envoyant une exception FatalErrorException.
void clear()
Supprime les éléments du tableau.
static ARCCORE_BASE_EXPORT std::optional< Int32 > tryParseFromEnvironment(StringView s, bool throw_if_invalid)
Informations pour le calcul du hash d'une donnée.
Informations de relecture des données.
Classe gérant un répertoire.
String file(const String &file_name) const override
Retourne le chemin complet du fichier file_name dans le répertoire.
Valeur retournée par un algorithme de hashage.
virtual void computeHash(DataHashInfo &hash_info)=0
Calcule le hash de la donnée.
Interface de lecture des données d'une variable (Version 2)
Interface de lecture des données d'une variable.
virtual Ref< ISerializedData > createSerializedDataRef(bool use_basic_type) const =0
Sérialise la donnée.
virtual void computeHash(IHashAlgorithm *algo, ByteArray &output) const =0
Calcul une clé de hashage sur cette donnée.
virtual IDataInternal * _commonInternal()=0
Interface d'un algorithme de hashage.
virtual IMeshFactoryMng * meshFactoryMng() const =0
Fabrique de maillages associée à ce gestionnaire.
virtual MeshHandle * findMeshHandle(const String &name, bool throw_exception)=0
Recherche le maillage de nom name.
virtual IParallelMng * parallelMng()=0
Gestionnaire de parallèlisme.
Interface du gestionnaire de parallélisme pour un sous-domaine.
virtual Int32 commRank() const =0
Rang de cette instance dans le communicateur.
virtual char reduce(eReduceType rt, char v)=0
Effectue la réduction de type rt sur le réel v et retourne la valeur.
Interface du gestionnaire d'un sous-domaine.
virtual IParallelMng * parallelMng()=0
Retourne le gestionnaire de parallélisme.
virtual IMeshMng * meshMng() const =0
Retourne le gestionnaire de maillage.
Interface du gestionnaire de traces.
Interface du gestionnaire de variables.
Interface d'une variable.
virtual String fullName() const =0
Nom complet de la variable (avec le préfixe de la famille)
@ PInShMem
Indique que la variable doit être alloué en mémoire partagée.
virtual IData * data()=0
Données associées à la variable.
static IXmlDocumentHolder * loadFromBuffer(Span< const Byte > buffer, const String &name, ITraceMng *tm)
Charge un document XML.
Gestion d'un document JSON.
void parse(Span< const Byte > bytes)
Lit le fichier au format UTF-8.
JSONValue root() const
Elément racine.
Représente une valeur JSON.
String value() const
Valeur sous forme de String. La chaîne retournée est nulle si 'null()' est vrai.
Int32 valueAsInt32() const
Valeur sous forme de Int64. Retourn 0 si 'null()' est vrai.
JSONValue expectedChild(StringView name) const
Valeur fille de nom name. Lance une exception si non trouvé.
JSONValue child(StringView name) const
Valeur fille de nom name. Retourne une valeur nulle si non trouvé.
Calcule la fonction de hashage MD5 d'un tableau.
IMesh * mesh() const
Maillage associé.
Exception lorsqu'une erreur fatale 'parallèle' est générée.
T * get() const
Retourne l'objet référé par l'instance.
Référence à une instance.
void reset()
Positionne l'instance au pointeur nul.
InstanceType * get() const
Instance associée ou nullptr si aucune.
Implémentation de l'algorithme SHA-1.
Ref< IHashAlgorithmContext > createContext() override
Créé un contexte pour calculer la valeur du hash de manière incrémentale.
Encapsulation d'un pointeur qui se détruit automatiquement.
constexpr __host__ __device__ pointer data() const noexcept
Pointeur sur le début de la vue.
constexpr __host__ __device__ SizeType size() const noexcept
Retourne la taille du tableau.
Vue d'un tableau d'éléments de type T.
Chaîne de caractères unicode.
bool null() const
Retourne true si la chaîne est nulle.
const char * localstr() const
Retourne la conversion de l'instance dans l'encodage UTF-8.
Span< const Byte > bytes() const
Retourne la conversion de l'instance dans l'encodage UTF-8.
bool empty() const
Vrai si la chaîne est vide (nulle ou "")
void split(StringContainer &str_array, char c) const
Découpe la chaîne suivant le caractère c.
Classe d'accès aux traces.
TraceAccessor(ITraceMng *m)
Construit un accesseur via le gestionnaire de trace m.
TraceMessage info() const
Flot pour un message d'information.
TraceMessage error() const
Flot pour un message d'erreur.
ITraceMng * traceMng() const
Gestionnaire de trace.
Paramètres nécessaires à la construction d'une variable.
Informations de relecture des données d'une variable.
Wrapper pour IDataReader2.
Interface pour utiliser IDataReader ou IDataReader2.
Wrapper pour IDataReader.
void _readMetaData(VariableMetaDataList &vmd_list, Span< const Byte > bytes)
Lecture des méta-données.
void _checkHashFunction(const VariableMetaDataList &vmd_list)
Vérifie les valeurs des fonctions de hashage.
Gestionnaire de variables.
XmlNode child(const String &name) const
Noeud fils de celui-ci de nom name.
String value() const
Valeur du noeud.
bool null() const
Vrai si le noeud est nul.
String toHexaString(ByteConstArrayView input)
Converti un tableau d'octet en sa représentation hexadécimale.
@ ReduceSum
Somme des valeurs.
-*- tab-width: 2; indent-tabs-mode: nil; coding: utf-8-with-signature -*-
Int32 Integer
Type représentant un entier.
UniqueArray< Byte > ByteUniqueArray
Tableau dynamique à une dimension de caractères.
Impl::SpanTypeFromSize< conststd::byte, SizeType >::SpanType asBytes(const SpanImpl< DataType, SizeType, Extent > &s)
Converti la vue en un tableau d'octets non modifiables.
auto makeRef(InstanceType *t) -> Ref< InstanceType >
Créé une référence sur un pointeur.
std::int32_t Int32
Type entier signé sur 32 bits.
auto makeRef(InstanceType *t) -> Ref< InstanceType >
Créé une référence sur un pointeur.