14#include "arcane/mesh/ItemsOwnerBuilder.h"
16#include "arcane/utils/PlatformUtils.h"
17#include "arcane/utils/SmallArray.h"
18#include "arcane/utils/HashTableMap2.h"
20#include "arcane/core/IParallelMng.h"
21#include "arcane/core/ParallelMngUtils.h"
22#include "arcane/core/IParallelExchanger.h"
23#include "arcane/core/ISerializeMessage.h"
24#include "arcane/core/ISerializer.h"
26#include "arcane/parallel/BitonicSortT.H"
28#include "arcane/mesh/ItemInternalMap.h"
29#include "arcane/mesh/NodeFamily.h"
30#include "arcane/mesh/EdgeFamily.h"
31#include "arcane/mesh/DynamicMesh.h"
33#include <unordered_set>
68class ItemsOwnerBuilderImpl
83 ItemOwnerInfo() =
default;
109 explicit ItemsOwnerBuilderImpl(
IMesh*
mesh);
113 void computeFacesOwner();
114 void computeEdgesOwner();
115 void computeNodesOwner();
120 Int32 m_verbose_level = 0;
147 explicit ItemOwnerInfoSortTraits(
bool use_cell_uid_to_sort)
165 if (m_use_cell_uid_to_sort) {
194 return pm->send(
ByteConstArrayView(messageSize(values),
reinterpret_cast<const Byte*
>(fsi_base)), rank,
false);
207 return ItemOwnerInfo(INT64_MAX, INT64_MAX, INT64_MAX, INT32_MAX, INT32_MAX);
216 bool m_use_cell_uid_to_sort =
true;
225ItemsOwnerBuilderImpl::
231 ARCANE_FATAL(
"Mesh is not an instance of 'DynamicMesh'");
234 m_verbose_level = v.value();
244void ItemsOwnerBuilderImpl::
247 m_items_owner_info.clear();
252 FaceFamily& face_family = m_mesh->trueFaceFamily();
254 info() <<
"** BEGIN ComputeFacesOwner nb_face=" << faces_map.
count();
271 info() <<
"ItemsOwnerBuilder: NB_FACE_TO_TRANSFER=" << faces_to_add.
size();
272 const Int32 verbose_level = m_verbose_level;
274 FaceInfoListView faces(&face_family);
275 for (Int32 lid : faces_to_add) {
276 Face face(faces[lid]);
279 if (verbose_level >= 2)
280 info() <<
"ADD lid=" << lid <<
" uid=" << face_uid <<
" cell_uid=" << cell.uniqueId() <<
" owner=" << cell.owner();
281 m_items_owner_info.add(ItemOwnerInfo(face_uid, face.
node(0).
uniqueId(), cell.uniqueId(), my_rank, cell.owner()));
288 _processSortedInfos(faces_map);
296void ItemsOwnerBuilderImpl::
299 m_items_owner_info.clear();
301 IParallelMng* pm = m_mesh->parallelMng();
303 ItemInternalMap& edges_map = m_mesh->edgesMap();
304 EdgeFamily& edge_family = m_mesh->trueEdgeFamily();
306 info() <<
"** BEGIN ComputeEdgesOwner nb_edge=" << edges_map.count();
311 UniqueArray<Int32> edges_to_add;
312 UniqueArray<Int64> edges_to_add_uid;
318 bool do_brute_force =
true;
319 edges_map.eachItem([&](Edge edge) {
320 Int32 nb_cell = edge.nbCell();
321 Int32 nb_cell_with_my_rank = 0;
322 for (
Cell cell : edge.cells())
323 if (cell.owner() == my_rank)
324 ++nb_cell_with_my_rank;
325 bool do_add = (nb_cell == 1 || (nb_cell != nb_cell_with_my_rank));
326 if (do_add || do_brute_force) {
327 edges_to_add.add(edge.localId());
328 edges_to_add_uid.add(edge.uniqueId());
331 edge.mutableItemBase().setOwner(my_rank, my_rank);
333 info() <<
"ItemsOwnerBuilder: NB_FACE_TO_TRANSFER=" << edges_to_add.size();
334 const Int32 verbose_level = m_verbose_level;
336 EdgeInfoListView edges(&edge_family);
337 for (
Int32 lid : edges_to_add) {
338 Edge edge(edges[lid]);
339 Int64 edge_uid = edge.uniqueId();
340 for (
Cell cell : edge.cells()) {
341 if (verbose_level >= 2)
342 info() <<
"ADD lid=" << lid <<
" uid=" << edge_uid <<
" cell_uid=" << cell.uniqueId() <<
" owner=" << cell.owner();
343 m_items_owner_info.add(
ItemOwnerInfo(edge_uid, edge.node(0).uniqueId(), cell.uniqueId(), my_rank, cell.owner()));
350 _processSortedInfos(edges_map);
352 edge_family.notifyItemsOwnerChanged();
358void ItemsOwnerBuilderImpl::
361 m_items_owner_info.clear();
363 IParallelMng* pm = m_mesh->parallelMng();
364 const Int32 my_rank = pm->commRank();
365 ItemInternalMap& faces_map = m_mesh->facesMap();
366 ItemInternalMap& nodes_map = m_mesh->nodesMap();
367 NodeFamily& node_family = m_mesh->trueNodeFamily();
369 info() <<
"** BEGIN ComputeNodesOwner nb_node=" << nodes_map.count();
372 nodes_map.eachItem([&](Node node) {
373 node.mutableItemBase().setOwner(my_rank, my_rank);
380 const Int32 verbose_level = m_verbose_level;
383 UniqueArray<Int32> nodes_to_add;
386 std::unordered_set<Int32> done_nodes;
388 const bool is_mono_dimension = m_mesh->meshKind().isMonoDimension();
390 FaceInfoListView faces(m_mesh->faceFamily());
391 if (is_mono_dimension) {
392 faces_map.eachItem([&](Face face) {
393 Int32 face_nb_cell = face.nbCell();
394 if (face_nb_cell == 2)
396 for (Node node : face.nodes()) {
397 Int32 node_id = node.localId();
398 if (done_nodes.find(node_id) == done_nodes.end()) {
399 nodes_to_add.add(node_id);
400 done_nodes.insert(node_id);
401 node.mutableItemBase().setOwner(A_NULL_RANK, my_rank);
413 ENUMERATE_ (Node, inode, m_mesh->allNodes()) {
414 nodes_to_add.add(inode.itemLocalId());
418 info() <<
"ItemsOwnerBuilder: NB_NODE_TO_ADD=" << nodes_to_add.size() <<
" is_mono_dim=" << is_mono_dimension;
419 NodeInfoListView nodes(&node_family);
420 for (
Int32 lid : nodes_to_add) {
421 Node node(nodes[lid]);
422 Int64 node_uid = node.uniqueId();
423 for (
Cell cell : node.cells()) {
424 if (verbose_level >= 2)
425 info() <<
"ADD lid=" << lid <<
" uid=" << node_uid <<
" cell_uid=" << cell.uniqueId() <<
" owner=" << cell.owner();
426 m_items_owner_info.add(
ItemOwnerInfo(node_uid, node_uid, cell.uniqueId(), my_rank, cell.owner()));
433 _processSortedInfos(nodes_map);
435 node_family.notifyItemsOwnerChanged();
448 const Int32 verbose_level = m_verbose_level;
451 items_sorter.setNeedIndexAndRank(
false);
453 items_sorter.
sort(m_items_owner_info);
455 m_items_owner_info = items_sorter.
keys();
456 info() <<
"END_ALL_ITEM_OWNER_SORTER time=" << (
Real)(sort_end_time - sort_begin_time);
457 if (verbose_level >= 2)
459 info() <<
"Sorted first_node_uid=" << x.m_first_node_uid <<
" item_uid="
460 << x.m_item_uid <<
" cell_uid=" << x.m_cell_uid <<
" owner=" << x.m_cell_owner;
467void ItemsOwnerBuilderImpl::
474 Int32 nb_sorted = items_owner_info.
size();
475 info() <<
"NbSorted=" << nb_sorted;
476 const bool is_last_rank = ((my_rank + 1) == nb_rank);
477 const bool is_first_rank = (my_rank == 0);
478 const Int32 verbose_level = m_verbose_level;
488 if (nb_sorted > 0 && !is_last_rank) {
489 Int32 send_index = nb_sorted;
490 Int64 last_uid = items_owner_info[nb_sorted - 1].m_item_uid;
491 for (
Int32 i = (nb_sorted - 1); i >= 0; --i) {
492 const ItemOwnerInfo& x = items_owner_info[i];
493 if (x.m_item_uid != last_uid) {
498 info() <<
"SendIndext=" << send_index <<
" nb_sorted=" << nb_sorted;
499 for (Int32 i = send_index; i < nb_sorted; ++i) {
500 const ItemOwnerInfo& x = items_owner_info[i];
501 items_owner_info_send_to_next.
add(x);
502 if (verbose_level >= 2)
503 info() <<
"AddSendToNext item_uid=" << x.m_item_uid <<
" owner=" << x.m_cell_owner
504 <<
" from_rank=" << x.m_item_sender_rank <<
" index=" << i;
507 Int32 nb_send_to_next = items_owner_info_send_to_next.
size();
508 info() <<
"NbSendToNext=" << nb_send_to_next;
510 Int32 nb_to_receive_from_previous = 0;
511 SmallArray<Parallel::Request> requests;
514 requests.add(pm->send(ConstArrayView<Int32>(1, &nb_send_to_next), my_rank + 1,
false));
516 requests.add(pm->
recv(ArrayView<Int32>(1, &nb_to_receive_from_previous), my_rank - 1,
false));
522 UniqueArray<ItemOwnerInfo> items_owner_info_received_from_previous(nb_to_receive_from_previous);
524 requests.add(ItemOwnerInfoSortTraits::send(pm, my_rank + 1, items_owner_info_send_to_next));
526 requests.add(ItemOwnerInfoSortTraits::recv(pm, my_rank - 1, items_owner_info_received_from_previous));
530 m_items_owner_info.resize(nb_sorted - nb_send_to_next);
531 items_owner_info = m_items_owner_info.view();
532 nb_sorted = items_owner_info.
size();
533 info() <<
"NbRemaining=" << nb_sorted;
535 Int64 current_item_uid = NULL_ITEM_UNIQUE_ID;
536 Int32 current_item_owner = A_NULL_RANK;
546 impl::HashTableMap2<Int32, UniqueArray<Int64>> resend_items_owner_info_map;
547 for (
Int32 index = 0; index < (nb_sorted + nb_to_receive_from_previous); ++index) {
551 if (index < nb_to_receive_from_previous)
552 first_ioi = &items_owner_info_received_from_previous[index];
554 first_ioi = &items_owner_info[index - nb_to_receive_from_previous];
555 Int64 item_uid = first_ioi->m_item_uid;
558 if (item_uid != current_item_uid) {
559 current_item_uid = item_uid;
561 current_item_owner = first_ioi->m_cell_owner;
563 current_item_owner = first_ioi->m_item_sender_rank;
564 if (verbose_level >= 2)
565 info() <<
"SetOwner from sorted index=" << index <<
" item_uid=" << current_item_uid <<
" new_owner=" << current_item_owner;
567 Int32 orig_sender = first_ioi->m_item_sender_rank;
568 UniqueArray<Int64>& send_array = resend_items_owner_info_map[orig_sender];
569 send_array.add(current_item_uid);
570 send_array.add(current_item_owner);
571 if (verbose_level >= 2)
572 info() <<
"SEND i=" << index <<
" rank=" << orig_sender <<
" item_uid=" << current_item_uid <<
" new_owner=" << current_item_owner;
576 info() <<
"NbResendRanks=" << resend_items_owner_info_map.size();
577 for (
const auto& [key, value] : resend_items_owner_info_map) {
578 if (verbose_level >= 1)
579 info() <<
"RESEND_INFO to_rank=" << key <<
" nb=" << value.size();
580 exchanger->addSender(key);
582 exchanger->initializeCommunicationsMessages();
585 for (
const auto& [key, value] : resend_items_owner_info_map) {
586 ISerializeMessage* sm = exchanger->messageToSend(index);
588 ISerializer* s = sm->serializer();
589 s->setMode(ISerializer::ModeReserve);
590 s->reserveArray(value);
596 exchanger->processExchange();
597 UniqueArray<Int64> receive_info;
599 for (
Integer i = 0, ns = exchanger->nbReceiver(); i < ns; ++i) {
600 ISerializeMessage* sm = exchanger->messageToReceive(i);
601 ISerializer* s = sm->serializer();
603 s->getArray(receive_info);
604 Int32 receive_size = receive_info.size();
605 if (verbose_level >= 1)
606 info() <<
"RECEIVE_INFO size=" << receive_size <<
" rank2=" << sm->destination();
608 if ((receive_size % 2) != 0)
609 ARCANE_FATAL(
"Size '{0}' is not a multiple of 2", receive_size);
610 Int32 buf_size = receive_size / 2;
611 for (
Int32 z = 0; z < buf_size; ++z) {
612 Int64 item_uid = receive_info[z * 2];
613 Int32 item_owner = CheckedConvert::toInt32(receive_info[(z * 2) + 1]);
614 impl::ItemBase x = items_map.
findItem(item_uid);
615 if (verbose_level >= 2)
616 info() <<
"SetOwner uid=" << item_uid <<
" new_owner=" << item_owner;
617 x.toMutable().setOwner(item_owner, my_rank);
629ItemsOwnerBuilder(IMesh* mesh)
630: m_p(std::make_unique<ItemsOwnerBuilderImpl>(mesh))
643void ItemsOwnerBuilder::
646 m_p->computeFacesOwner();
652void ItemsOwnerBuilder::
655 m_p->computeEdgesOwner();
661void ItemsOwnerBuilder::
664 m_p->computeNodesOwner();
#define ARCANE_FATAL(...)
Macro throwing a FatalErrorException.
Integer size() const
Number of elements in the vector.
Modifiable view of an array of type T.
constexpr const_pointer data() const noexcept
Pointer to the start of the view.
void add(ConstReferenceType val)
Adds element val to the end of the array.
Constant view of an array of type T.
constexpr const_pointer data() const noexcept
Pointer to the allocated memory.
constexpr Integer size() const noexcept
Number of elements in the array.
Template class for converting a type.
Cell cell(Int32 i) const
i-th cell of the face
Int32 nbCell() const
Number of cells of the face (1 or 2).
CellConnectedListViewType cells() const
List of cells of the face.
Interface of the parallelism manager for a subdomain.
virtual Int32 commRank() const =0
Rank of this instance in the communicator.
virtual void recv(ArrayView< char > values, Int32 rank)=0
virtual Int32 commSize() const =0
Number of instances in the communicator.
virtual void waitAllRequests(ArrayView< Request > rvalues)=0
Blocks while waiting for the rvalues requests to complete.
@ ModePut
The serializer expects reserve().
@ ModeGet
The serializer expects get().
Node node(Int32 i) const
i-th node of the entity
impl::MutableItemBase mutableItemBase() const
Mutable internal part of the entity.
constexpr Int32 localId() const
Local identifier of the entity in the processor subdomain.
Int32 owner() const
Owner subdomain number of the entity.
ItemUniqueId uniqueId() const
Unique identifier across all domains.
void setOwner(Integer suid, Int32 current_sub_domain)
Sets the sub-domain number of the entity owner.
Parallel bitonic sort algorithm.
ConstArrayView< KeyType > keys() const override
After a sort, returns the list of elements on this rank.
void sort(ConstArrayView< KeyType > keys) override
Parallelly sorts the elements of keys on all ranks.
TraceAccessor(ITraceMng *m)
Constructs an accessor via the trace manager m.
TraceMessage info() const
Flow for an information message.
ITraceMng * traceMng() const
Trace manager.
1D data vector with value semantics (STL style).
Implementation of a mesh.
IParallelMng * parallelMng() override
Parallelism manager.
void notifyItemsOwnerChanged() override
Notifies that the entities specific to the family's subdomain have been modified.
Associative array of ItemInternal.
impl::ItemBase findItem(Int64 uid) const
Returns the unique ID entity uid.
void eachItem(const Lambda &lambda)
Template function to iterate over the instance's entities.
Int32 count() const
Number of elements in the table.
Information about a shared entity.
Int32 m_item_sender_rank
rank of the one who created this instance
Int64 m_first_node_uid
uniqueId() of the first node of the entity
Int32 m_cell_owner
Owner of the cell connected to this entity.
Int64 m_item_uid
uniqueId() of the entity
Int64 m_cell_uid
uniqueId() of the cell to which the entity belongs
bool m_use_cell_uid_to_sort
Indicates how to perform the sort.
void _sortInfos()
Sorts the instances contained in m_items_owner_info and replaces the sorted values in this same array...
Ref< IParallelExchanger > createExchangerRef(IParallelMng *pm)
Returns an interface to transfer messages between ranks.
ArrayView< Byte > ByteArrayView
C equivalent of a 1D array of characters.
std::int64_t Int64
Signed integer type of 64 bits.
Int32 Integer
Type representing an integer.
double Real
Type representing a real number.
ConstArrayView< Byte > ByteConstArrayView
C equivalent of a 1D array of characters.
unsigned char Byte
Type of a byte.
@ Cell
The mesh is AMR by cell.
std::int32_t Int32
Signed integer type of 32 bits.