additional documentation, make name changes more consistent

include/openmc/message_passing.h

@@ -13,7 +13,7 @@ namespace mpi {
   extern bool master;
 
 #ifdef OPENMC_MPI
-  extern MPI_Datatype bank;
+  extern MPI_Datatype source_site;
   extern MPI_Comm intracomm;
 #endif
 

include/openmc/particle.h

@@ -21,11 +21,12 @@ namespace openmc {
 // Forward declare the Surface class for use in Particle::cross_vacuum_bc, etc.
 class Surface;
 
-//============================================================================
-//! State of a particle being transported through geometry
-//! This class defines actions particles can take. Its base
-//! class defines particle data layout in memory.
-//============================================================================
+/*
+ * The Particle class encompasses data and methods for transporting particles
+ * through their lifecycle. Its base class defines particle data layout in
+ * memory. A more detailed description of the rationale behind this approach
+ * can be found in particle_data.h.
+ */
 
 class Particle : public ParticleData {
 public:

include/openmc/particle_data.h

@@ -164,6 +164,32 @@ struct BoundaryInfo {
 //! Defines how particle data is laid out in memory
 //============================================================================
 
+/*
+ * This class was added in order to separate the layout and access of particle
+ * data from particle physics operations during a development effort to get
+ * OpenMC running on GPUs. In the event-based Monte Carlo method, one creates
+ * an array of particles on which actions like cross section lookup and surface
+ * crossing are done en masse, which works best on vector computers of yore and
+ * modern GPUs. It has been shown in the below publication that arranging
+ * particle data into a structure of arrays rather than an array of structures
+ * enhances performance on GPUs. For instance, rather than having an
+ * std::vector<Particle> where consecutive particle energies would be separated
+ * by about 400 bytes, one would create a structure which has a single
+ * std::vector<double> of energies.  The motivation here is that more coalesced
+ * memory accesses occur, in the parlance of GPU programming.
+ *
+ * So, this class enables switching between the array-of-structures and
+ * structure- of-array data layout at compile time. In GPU branches of the
+ * code, our Particle class inherits from a class that provides an array of
+ * particle energies, and can access them using the E() method (defined below).
+ * In the CPU code, we inherit from this class which gives the conventional
+ * layout of particle data, useful for history-based tracking.
+ *
+ * As a result, we always use the E(), r_last(), etc. methods to access
+ * particle data in order to keep a unified interface between
+ * structure-of-array and array-of-structure code on either CPU or GPU code
+ * while sharing the same physics code on each codebase.
+ */
 class ParticleData {
 
 public:

openmc/lib/core.py

@@ -11,7 +11,7 @@
 import openmc.lib
 
 
-class _Bank(Structure):
+class _SourceSite(Structure):
     _fields_ = [('r', c_double*3),
                 ('u', c_double*3),
                 ('E', c_double),
@@ -73,7 +73,7 @@ class _Bank(Structure):
                            c_double]
 _dll.openmc_run_linsolver.argtypes = _run_linsolver_argtypes
 _dll.openmc_run_linsolver.restype = c_int
-_dll.openmc_source_bank.argtypes = [POINTER(POINTER(_Bank)), POINTER(c_int64)]
+_dll.openmc_source_bank.argtypes = [POINTER(POINTER(_SourceSite)), POINTER(c_int64)]
 _dll.openmc_source_bank.restype = c_int
 _dll.openmc_source_bank.errcheck = _error_handler
 _dll.openmc_simulation_init.restype = c_int
@@ -342,13 +342,13 @@ def source_bank():
 
     """
     # Get pointer to source bank
-    ptr = POINTER(_Bank)()
+    ptr = POINTER(_SourceSite)()
     n = c_int64()
     _dll.openmc_source_bank(ptr, n)
 
     try:
         # Convert to numpy array with appropriate datatype
-        bank_dtype = np.dtype(_Bank)
+        bank_dtype = np.dtype(_SourceSite)
         return as_array(ptr, (n.value,)).view(bank_dtype)
 
     except ValueError as err:

src/eigenvalue.cpp

@@ -230,7 +230,7 @@ void synchronize_bank()
       // process
       if (neighbor != mpi::rank) {
         requests.emplace_back();
-        MPI_Isend(&temp_sites[index_local], static_cast<int>(n), mpi::bank,
+        MPI_Isend(&temp_sites[index_local], static_cast<int>(n), mpi::source_site,
           neighbor, mpi::rank, mpi::intracomm, &requests.back());
       }
 
@@ -276,7 +276,7 @@ void synchronize_bank()
       // asynchronous receive for the source sites
 
       requests.emplace_back();
-      MPI_Irecv(&simulation::source_bank[index_local], static_cast<int>(n), mpi::bank,
+      MPI_Irecv(&simulation::source_bank[index_local], static_cast<int>(n), mpi::source_site,
             neighbor, neighbor, mpi::intracomm, &requests.back());
 
     } else {

src/finalize.cpp

@@ -143,7 +143,7 @@ int openmc_finalize()
 
   // Free all MPI types
 #ifdef OPENMC_MPI
-  if (mpi::bank != MPI_DATATYPE_NULL) MPI_Type_free(&mpi::bank);
+  if (mpi::source_site != MPI_DATATYPE_NULL) MPI_Type_free(&mpi::source_site);
 #endif
 
   return 0;

src/initialize.cpp

@@ -151,8 +151,8 @@ void initialize_mpi(MPI_Comm intracomm)
 
   int blocks[] {3, 3, 1, 1, 1, 1, 1, 1, 1};
   MPI_Datatype types[] {MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_INT, MPI_INT, MPI_INT, MPI_LONG, MPI_LONG};
-  MPI_Type_create_struct(9, blocks, disp, types, &mpi::bank);
-  MPI_Type_commit(&mpi::bank);
+  MPI_Type_create_struct(9, blocks, disp, types, &mpi::source_site);
+  MPI_Type_commit(&mpi::source_site);
 }
 #endif // OPENMC_MPI
 

src/message_passing.cpp

@@ -9,7 +9,7 @@ bool master {true};
 
 #ifdef OPENMC_MPI
 MPI_Comm intracomm {MPI_COMM_NULL};
-MPI_Datatype bank {MPI_DATATYPE_NULL};
+MPI_Datatype source_site {MPI_DATATYPE_NULL};
 #endif
 
 extern "C" bool openmc_master() { return mpi::master; }

src/state_point.cpp

@@ -666,7 +666,7 @@ write_source_bank(hid_t group_id, bool surf_source_bank)
 #ifdef OPENMC_MPI
       // Receive source sites from other processes
       if (i > 0)
-        MPI_Recv(source_bank->data(), count[0], mpi::bank, i, i,
+        MPI_Recv(source_bank->data(), count[0], mpi::source_site, i, i,
                  mpi::intracomm, MPI_STATUS_IGNORE);
 #endif
 
@@ -691,7 +691,7 @@ write_source_bank(hid_t group_id, bool surf_source_bank)
 #endif
   } else {
 #ifdef OPENMC_MPI
-    MPI_Send(source_bank->data(), count_size, mpi::bank,
+    MPI_Send(source_bank->data(), count_size, mpi::source_site,
       0, mpi::rank, mpi::intracomm);
 #endif
   }