Feature: boundary condition handler MPI communication

src/t8.h

@@ -161,6 +161,7 @@ typedef enum {
   T8_MPI_CMESH_UNIFORM_BOUNDS_END,      /**< Used for cmesh uniform bounds computation. */
   T8_MPI_TEST_ELEMENT_PACK_TAG,         /**< Used for testing mpi pack and unpack functionality */
   T8_MPI_PFC_TAG,                       /**< Used for data exchange during partition for coarsening. */
+  T8_MPI_BOUNDARY_CONDITION_SYNC_TAG,   /**< Used for the global synchronization of boundary conditions. */
   T8_MPI_TAG_LAST                       /**< Dummy last MPI tag. */
 } t8_MPI_tag_t;
 

src/t8_cmesh/t8_cmesh.cxx

@@ -686,7 +686,7 @@ t8_cmesh_bcast (const t8_cmesh_t cmesh_in, const int root, sc_MPI_Comm comm)
     t8_cmesh_struct_t cmesh;
     t8_gloidx_t num_trees_per_eclass[T8_ECLASS_COUNT];
     size_t stash_elem_counts[3];
-    int pre_commit; /* True, if cmesh on root is not committed yet. */
+    int pre_commit; /** True if cmesh on root is not committed yet. */
 #if T8_ENABLE_DEBUG
     sc_MPI_Comm comm;
 #endif
@@ -768,6 +768,9 @@ t8_cmesh_bcast (const t8_cmesh_t cmesh_in, const int root, sc_MPI_Comm comm)
     if (meta_info.cmesh.profile != nullptr) {
       t8_cmesh_set_profiling (cmesh_in, 1);
     }
+    if (meta_info.cmesh.boundary_condition_handler != nullptr) {
+      t8_cmesh_add_boundary_condition_handler (cmesh_out);
+    }
     for (iclass = 0; iclass < T8_ECLASS_COUNT; iclass++) {
       cmesh_out->num_trees_per_eclass[iclass] = meta_info.num_trees_per_eclass[iclass];
       cmesh_out->num_local_trees_per_eclass[iclass] = meta_info.num_trees_per_eclass[iclass];
@@ -793,6 +796,10 @@ t8_cmesh_bcast (const t8_cmesh_t cmesh_in, const int root, sc_MPI_Comm comm)
       cmesh_out->committed = 1;
     }
   }
+  /* Broadcast boundary conditions if set */
+  if (cmesh_out->boundary_condition_handler != nullptr) {
+    cmesh_out->boundary_condition_handler->bcast (root, comm);
+  }
 
   cmesh_out->mpirank = mpirank;
   cmesh_out->mpisize = mpisize;

src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx

@@ -25,11 +25,14 @@
  * Implementation context of \ref t8_cmesh_boundary_condition_handler.hxx
  */
 
+#include <t8.h>
 #include <t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx>
 #include <t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler_types.h>
 #include <t8_cmesh/t8_cmesh_internal/t8_cmesh_types.h>
 #include <t8_cmesh/t8_cmesh_internal/t8_cmesh_stash.h>
 
+#include <cstring>
+
 using namespace detail;
 
 #if T8_ENABLE_DEBUG
@@ -91,3 +94,103 @@ t8_cmesh_boundary_condition_handler::get_boundary_condition_attribute_key () con
 {
   return T8_CMESH_BOUNDARY_CONDITION_ATTRIBUTE_KEY;
 }
+
+std::vector<char>
+t8_cmesh_boundary_condition_handler::serialize_map () const
+{
+  std::vector<char> serial_data;
+  /* Fill serial_data with strings only. The hashes can be re-generated locally. */
+  for (const auto &[key, string] : t8_cmesh_boundary_condition_handler::m_boundary_conditions) {
+    serial_data.insert (serial_data.end (), string.begin (), string.end ());
+    /* Null terminate strings to be able to split them again later. */
+    serial_data.push_back ('\0');
+  }
+  serial_data.shrink_to_fit ();
+  return serial_data;
+}
+
+void
+t8_cmesh_boundary_condition_handler::unpack_map (std::vector<char> &serial_data, bool overwrite)
+{
+  if (overwrite) {
+    m_boundary_conditions.clear ();
+  }
+
+  /* Iterate over stings. */
+  for (const char *string = serial_data.data (); string < serial_data.data () + serial_data.size ();
+       string += std::strlen (string) + 1) {
+    /* Interpret c string as std::string, rehash it and insert it. */
+    std::string value (string);
+    t8_cmesh_boundary_condition_handler::m_boundary_conditions.try_emplace (
+      t8_cmesh_boundary_condition_handler::hash_boundary_condition_name (value), std::move (value));
+  }
+}
+
+void
+t8_cmesh_boundary_condition_handler::synchronize (sc_MPI_Comm comm)
+{
+  T8_ASSERT (comm != sc_MPI_COMM_NULL);
+
+  int rank, mpisize;
+  sc_MPI_Comm_rank (comm, &rank);
+  sc_MPI_Comm_size (comm, &mpisize);
+
+  if (mpisize == 1) {
+    /* Nothing to do. */
+    return;
+  }
+
+  /* Prepare the sendbuffer. */
+  std::vector<char> send_buffer = t8_cmesh_boundary_condition_handler::serialize_map ();
+
+  /* Communicate the local sizes */
+  int local_size = static_cast<int> (send_buffer.size ());
+  std::vector<int> sizes (mpisize);
+  sc_MPI_Allgather (&local_size, 1, sc_MPI_INT, sizes.data (), 1, sc_MPI_INT, comm);
+
+  /* Compute the offsets for the data and create receive buffer. */
+  std::vector<int> offsets (mpisize);
+  int total_size = 0;
+  for (int rank = 0; rank < mpisize; ++rank) {
+    offsets[rank] = total_size;
+    total_size += sizes[rank];
+  }
+  std::vector<char> recv_buffer (total_size);
+
+  /* Communicate. */
+  sc_MPI_Allgatherv (send_buffer.data (), local_size, sc_MPI_BYTE, recv_buffer.data (), sizes.data (), offsets.data (),
+                     sc_MPI_BYTE, comm);
+
+  /* Unpack the data. */
+  t8_cmesh_boundary_condition_handler::unpack_map (recv_buffer, false);
+}
+
+void
+t8_cmesh_boundary_condition_handler::bcast (int main_rank, sc_MPI_Comm comm)
+{
+  int rank;
+  sc_MPI_Comm_rank (comm, &rank);
+
+  /* Buffer for sending and receiving */
+  std::vector<char> buffer;
+
+  /* Serialize data. */
+  if (rank == main_rank) {
+    buffer = t8_cmesh_boundary_condition_handler::serialize_map ();
+  }
+
+  /* Prepare buffer length. */
+  int size = static_cast<int> (buffer.size ());
+  sc_MPI_Bcast (&size, 1, sc_MPI_INT, main_rank, comm);
+  if (rank != main_rank) {
+    buffer.resize (size);
+  }
+
+  /* Broadcast data */
+  sc_MPI_Bcast (buffer.data (), size, sc_MPI_BYTE, main_rank, comm);
+
+  /* Add data to local map. */
+  if (rank != main_rank) {
+    t8_cmesh_boundary_condition_handler::unpack_map (buffer, true);
+  }
+}

src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx

@@ -73,6 +73,8 @@ struct t8_cmesh_boundary_condition_handler
   using boundary_condition_hash = T8Type<size_t, boundary_condition_hash_tag, EqualityComparable>;
 
  public:
+  /**************************************** CONSTRUCTORS & ASSIGNMENT OPERATORS ****************************************/
+
   /**
    * Standard constructor. Associates the handler with a cmesh
    * \param [in] cmesh
@@ -81,6 +83,42 @@ struct t8_cmesh_boundary_condition_handler
   {
   }
 
+  /**
+   * Copy constructor.
+   * \param [in]  other  The other.
+   */
+  t8_cmesh_boundary_condition_handler (const t8_cmesh_boundary_condition_handler &other) = default;
+
+  /**
+   * Move constructor.
+   * \param [in]  other  The other.
+   */
+  t8_cmesh_boundary_condition_handler (t8_cmesh_boundary_condition_handler &&other) noexcept = default;
+
+  /**
+   * Copy assignment operator.
+   * \param [in]  other  The other.
+   * \return      A copy of this.
+   */
+  t8_cmesh_boundary_condition_handler &
+  operator= (const t8_cmesh_boundary_condition_handler &other)
+    = default;
+
+  /**
+   * Move assignment operator.
+   * \param [in]  other  The other.
+   * \return      A reference to a moved version of this.
+   */
+  t8_cmesh_boundary_condition_handler &
+  operator= (t8_cmesh_boundary_condition_handler &&other) noexcept
+    = default;
+
+  /**
+   * The destructor.
+   */
+  ~t8_cmesh_boundary_condition_handler () = default;
+
+  /**************************************** BOUNDARY CONDITION SETUP ****************************************/
   /**
    * Applies boundary conditions to the faces of a cmesh cell.
    *
@@ -90,21 +128,70 @@ struct t8_cmesh_boundary_condition_handler
    *                                  as the eclass of the cell has faces.
    */
   template <std::ranges::input_range TStringRange>
-    requires std::convertible_to<std::ranges::range_reference_t<TStringRange>, std::string_view>
+    requires std::convertible_to<std::ranges::range_value_t<TStringRange>, std::string_view>
   inline void
   add_boundary_conditions (t8_gloidx_t gtreeid, TStringRange boundary_conditions)
   {
     std::vector<boundary_condition_hash> hashes;
     hashes.reserve (std::size (boundary_conditions));
     for (const auto &boundary_condition : boundary_conditions) {
-      const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition);
-      m_boundary_conditions.try_emplace (hash, boundary_condition);
+      const std::string_view boundary_condition_view = boundary_condition;
+      const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition_view);
+      [[maybe_unused]] const auto inserted = m_boundary_conditions.try_emplace (hash, boundary_condition_view);
+#if T8_ENABLE_DEBUG
+      if (inserted.second) {
+        t8_debugf ("Registered boundary condition %.*s\n", static_cast<int> (boundary_condition_view.size ()),
+                   boundary_condition_view.data ());
+      }
+#endif
       hashes.emplace_back (std::move (hash));
     }
     t8_cmesh_set_attribute (m_cmesh, gtreeid, t8_get_package_id (), get_boundary_condition_attribute_key (),
                             hashes.data (), sizeof (boundary_condition_hash) * hashes.size (), 0);
   }
 
+  /**
+   * Adds a boundary condition name to this handler without registering it to a tree.
+   * Mostly needed for debugging and testing reasons.
+   * Boundary conditions added via \ref add_boundary_conditions do not need to be registered explicitly.
+   * \tparam TString                  A string-like object.
+   * \param [in]  boundary_condition  The name of the boundary condition.
+   */
+  template <typename TString>
+    requires std::convertible_to<TString, std::string_view>
+  inline void
+  register_boundary_condition (TString &&boundary_condition)
+  {
+    const std::string boundary_condition_string { boundary_condition };
+    const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition_string);
+
+    [[maybe_unused]] const auto inserted
+      = m_boundary_conditions.try_emplace (hash, std::move (boundary_condition_string));
+
+#if T8_ENABLE_DEBUG
+    if (inserted.second) {
+      const std::string_view boundary_condition_view = boundary_condition;
+      t8_debugf ("Registered boundary condition %.*s\n", static_cast<int> (boundary_condition_view.size ()),
+                 boundary_condition_view.data ());
+    }
+#endif
+  }
+
+  /**
+   * Updates the internal cmesh. The boundary condition handler can only be given to uncommitted cmeshes.
+   * \param [in] new_cmesh  The new cmesh.
+   */
+  inline void
+  set_cmesh (t8_cmesh_t new_cmesh)
+  {
+    T8_ASSERT (t8_cmesh_is_initialized (new_cmesh));
+    T8_ASSERTF (t8_cmesh_is_committed (new_cmesh, 0),
+                "The boundary condition handler can only be set for uncommitted cmeshes.\n");
+    m_cmesh = new_cmesh;
+  }
+
+  /**************************************** BOUNDARY CONDITION RETRIEVAL ****************************************/
+
   /**
    * Retrieves the boundary conditions of a cmesh cell.
    *
@@ -195,6 +282,23 @@ struct t8_cmesh_boundary_condition_handler
     return std::nullopt;
   }
 
+  /**
+   * Get all registered boundary condition names.
+   * \return A vector containing all registered boundary condition names.
+   */
+  inline std::vector<std::string_view>
+  get_registered_boundary_conditions () const
+  {
+    std::vector<std::string_view> boundary_conditions;
+    boundary_conditions.reserve (m_boundary_conditions.size ());
+    for (const auto &boundary_condition : m_boundary_conditions) {
+      boundary_conditions.push_back (boundary_condition.second);
+    }
+    return boundary_conditions;
+  }
+
+  /**************************************** HELPER FUNCTIONS ****************************************/
+
 #if T8_ENABLE_DEBUG
   /** Verifies the proper attribution of boundary conditions. Can only be called on a cmesh
    * during commit.
@@ -234,9 +338,9 @@ struct t8_cmesh_boundary_condition_handler
    * \return                              The hash of the name.
    */
   inline boundary_condition_hash
-  hash_boundary_condition_name (const std::string &boundary_condition_name) const
+  hash_boundary_condition_name (const std::string_view &boundary_condition_name) const
   {
-    return boundary_condition_hash (std::hash<std::string> {}(boundary_condition_name));
+    return boundary_condition_hash (std::hash<std::string_view> {}(boundary_condition_name));
   }
 
   /**
@@ -267,6 +371,45 @@ struct t8_cmesh_boundary_condition_handler
     return m_boundary_conditions.at (hash);
   }
 
+  /**************************************** MPI HELPER FUNCTIONS ****************************************/
+
+ public:
+  /**
+   * Synchronizes the contents of the boundary condition handler across all processes.
+   * \param [in]  comm      The communicator to use.
+   */
+  void
+  synchronize (sc_MPI_Comm comm);
+
+  /**
+   * Broadcasts the boundary conditions from \a main_rank to all other ranks.
+   * \param [in]  main_rank The main rank from which to broadcast.
+   * \param [in]  comm      The communicator to use.
+   */
+  void
+  bcast (int main_rank, sc_MPI_Comm comm);
+
+  /**
+   * Converts the contents of m_boundary_conditions into a serial vector of chars.
+   * The keys are omitted and only the strings are serialized.
+   * In the serialized vector, the individual strings are null-terminated.
+   * \return The serialized map.
+   */
+  std::vector<char>
+  serialize_map () const;
+
+  /**
+   * Unpacks and integrates the \a serial_data into m_boundary_conditions.
+   * It either merges the already existing data or overwrites the complete map if \a overwrite is set to true.
+   * \param [in]  serial_data   The data to unpack and integrate.
+   * \param [in]  overwrite     Overwrites the data in this handler if true. Merges the data with the existing data on false.
+   */
+  void
+  unpack_map (std::vector<char> &serial_data, bool overwrite);
+
+  /**************************************** MEMBERS ****************************************/
+
+ private:
   /** The associated cmesh of this struct */
   t8_cmesh_t m_cmesh;
 

src/t8_cmesh/t8_cmesh_boundary_conditions/t8_cmesh_boundary_conditions.hxx

@@ -45,7 +45,7 @@
  *                                  as the eclass of the cell has faces.
  */
 template <std::ranges::input_range TStringRange>
-  requires std::convertible_to<std::ranges::range_reference_t<TStringRange>, std::string_view>
+  requires std::convertible_to<std::ranges::range_value_t<TStringRange>, std::string_view>
 void
 t8_cmesh_set_boundary_conditions (t8_cmesh_t cmesh, t8_gloidx_t gtreeid, TStringRange boundary_conditions)
 {