MOM_domains.F90

Go to the documentation of this file.

module mom_domains

!***********************************************************************
!*                   GNU General Public License                        *
!* This file is a part of MOM.                                         *
!*                                                                     *
!* MOM is free software; you can redistribute it and/or modify it and  *
!* are expected to follow the terms of the GNU General Public License  *
!* as published by the Free Software Foundation; either version 2 of   *
!* the License, or (at your option) any later version.                 *
!*                                                                     *
!* MOM is distributed in the hope that it will be useful, but WITHOUT  *
!* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  *
!* or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public    *
!* License for more details.                                           *
!*                                                                     *
!* For the full text of the GNU General Public License,                *
!* write to: Free Software Foundation, Inc.,                           *
!*           675 Mass Ave, Cambridge, MA 02139, USA.                   *
!* or see:   http://www.gnu.org/licenses/gpl.html                      *
!***********************************************************************

use mom_coms, only : pe_here, root_pe, num_pes, mom_infra_init, mom_infra_end
use mom_coms, only : broadcast, sum_across_pes, min_across_pes, max_across_pes
use mom_error_handler, only : mom_error, mom_mesg, note, warning, fatal, is_root_pe
use mom_file_parser, only : get_param, log_param, log_version
use mom_file_parser, only : param_file_type
use mom_string_functions, only : slasher

use mpp_domains_mod, only : mpp_define_layout, mpp_get_boundary
use mpp_domains_mod, only : mom_define_io_domain => mpp_define_io_domain
use mpp_domains_mod, only : mom_define_domain => mpp_define_domains
use mpp_domains_mod, only : domain2d, domain1d, mpp_get_data_domain
use mpp_domains_mod, only : mpp_get_compute_domain, mpp_get_global_domain
use mpp_domains_mod, only : global_field_sum => mpp_global_sum
use mpp_domains_mod, only : mpp_update_domains, cyclic_global_domain, fold_north_edge
use mpp_domains_mod, only : mpp_start_update_domains, mpp_complete_update_domains
use mpp_domains_mod, only : mpp_create_group_update, mpp_do_group_update
use mpp_domains_mod, only : group_pass_type => mpp_group_update_type
use mpp_domains_mod, only : mpp_reset_group_update_field
use mpp_domains_mod, only : mpp_group_update_initialized
use mpp_domains_mod, only : mpp_start_group_update, mpp_complete_group_update
use mpp_domains_mod, only : compute_block_extent => mpp_compute_block_extent
use mpp_parameter_mod, only : agrid, bgrid_ne, cgrid_ne, scalar_pair, bitwise_exact_sum, corner
use mpp_parameter_mod, only : to_east => wupdate, to_west => eupdate, omit_corners => edgeupdate
use mpp_parameter_mod, only : to_north => supdate, to_south => nupdate
use fms_io_mod,        only : file_exist, parse_mask_table

implicit none ; private

! #include <MOM_memory.h>

public :: mom_domains_init, mom_infra_init, mom_infra_end, get_domain_extent
public :: mom_define_domain, mom_define_io_domain, clone_mom_domain
public :: pass_var, pass_vector, broadcast, pe_here, root_pe, num_pes
public :: pass_var_start, pass_var_complete, fill_symmetric_edges
public :: pass_vector_start, pass_vector_complete
public :: global_field_sum, sum_across_pes, min_across_pes, max_across_pes
public :: agrid, bgrid_ne, cgrid_ne, scalar_pair, bitwise_exact_sum, corner
public :: to_east, to_west, to_north, to_south, to_all, omit_corners
public :: create_group_pass, do_group_pass, group_pass_type
public :: start_group_pass, complete_group_pass
public :: compute_block_extent

interface pass_var
  module procedure pass_var_3d, pass_var_2d
end interface pass_var

interface pass_vector
  module procedure pass_vector_3d, pass_vector_2d
end interface pass_vector

interface pass_var_start
  module procedure pass_var_start_3d, pass_var_start_2d
end interface pass_var_start

interface pass_var_complete
  module procedure pass_var_complete_3d, pass_var_complete_2d
end interface pass_var_complete

interface pass_vector_start
  module procedure pass_vector_start_3d, pass_vector_start_2d
end interface pass_vector_start

interface pass_vector_complete
  module procedure pass_vector_complete_3d, pass_vector_complete_2d
end interface pass_vector_complete

interface create_group_pass
  module procedure create_var_group_pass_2d
  module procedure create_var_group_pass_3d
  module procedure create_vector_group_pass_2d
  module procedure create_vector_group_pass_3d
end interface create_group_pass

interface fill_symmetric_edges
  module procedure fill_vector_symmetric_edges_2d !, fill_vector_symmetric_edges_3d
!   module procedure fill_scalar_symmetric_edges_2d, fill_scalar_symmetric_edges_3d
end interface fill_symmetric_edges

interface clone_mom_domain
  module procedure clone_md_to_md, clone_md_to_d2d
end interface clone_mom_domain

!> The MOM_domain_type contains information about the domain decompositoin.
type, public :: mom_domain_type
  type(domain2d), pointer :: mpp_domain => null() !< The FMS domain with halos
                                !! on this processor, centered at h points.
  integer :: niglobal           !< The total horizontal i-domain size.
  integer :: njglobal           !< The total horizontal j-domain size.
  integer :: nihalo             !< The i-halo size in memory.
  integer :: njhalo             !< The j-halo size in memory.
  logical :: symmetric          !< True if symmetric memory is used with
                                !! this domain.
  logical :: nonblocking_updates  !< If true, non-blocking halo updates are
                                !! allowed.  The default is .false. (for now).
  logical :: thin_halo_updates  !< If true, optional arguments may be used to
                                !! specify the width of the halos that are
                                !! updated with each call.
  integer :: layout(2)          !< This domain's processor layout.  This is
                                !! saved to enable the construction of related
                                !! new domains with different resolutions or
                                !! other properties.
  integer :: io_layout(2)       !< The IO-layout used with this domain.
  integer :: x_flags            !< Flag that specifies the properties of the
                                !! domain in the i-direction in a define_domain call.
  integer :: y_flags            !< Flag that specifies the properties of the
                                !! domain in the j-direction in a define_domain call.
  logical :: use_io_layout      !< True if an I/O layout is available.
  logical, pointer :: maskmap(:,:) => null() !< A pointer to an array indicating
                                !! which logical processors are actually used for
                                !! the ocean code. The other logical processors
                                !! would be contain only land points and are not
                                !! assigned to actual processors. This need not be
                                !! assigned if all logical processors are used.
end type mom_domain_type

integer, parameter :: to_all = to_east + to_west + to_north + to_south

contains

! #@# This subroutine needs a doxygen description
subroutine pass_var_3d(array, MOM_dom, sideflag, complete, position, halo)
  real, dimension(:,:,:), intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! sothe halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                                    !! halo updates should be completed before
                                                    !! progress resumes. Omitting complete is the
                                                    !! same as setting complete to .true.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER by
                                                    !! default.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in,opt)  complete - An optional argument indicating whether the halo updates
!                       should be completed before progress resumes.  Omitting
!                       complete is the same as setting complete to .true.
!  (in,opt)   position - An optional argument indicating the position.  This is
!                       usally CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: dirflag
  logical :: block_til_complete

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif
  block_til_complete = .true.
  if (present(complete)) block_til_complete = complete

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_update_domains(array, mom_dom%mpp_domain, flags=dirflag, &
                        complete=block_til_complete, position=position, &
                        whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_update_domains(array, mom_dom%mpp_domain, flags=dirflag, &
                          complete=block_til_complete, position=position)
  endif

end subroutine pass_var_3d

! #@# This subroutine needs a doxygen description
subroutine pass_var_2d(array, MOM_dom, sideflag, complete, position, halo)
  real, dimension(:,:),  intent(inout) :: array    !< The array which is having its halos points
                                                   !! exchanged.
  type(mom_domain_type), intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                   !! needed to determine where data should be sent.
  integer,     optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  logical,     optional, intent(in)    :: complete !< An optional argument indicating whether the
                                                   !! halo updates should be completed before
                                                   !! progress resumes.  Omitting complete is the
                                                   !! same as setting complete to .true.
  integer,     optional, intent(in)    :: position !< An optional argument indicating the position.
                                                   !!  This is usally CORNER, but is CENTER
                                                   !! by default.
  integer,     optional, intent(in)    :: halo     !< The size of the halo to update - the full halo
                                                   !! by default.
! Arguments: array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in,opt)  complete - An optional argument indicating whether the halo updates
!                       should be completed before progress resumes.  Omitting
!                       complete is the same as setting complete to .true.
!  (in,opt)  position - An optional argument indicating the position.  This is
!                       usally CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.

  integer :: dirflag
  logical :: block_til_complete

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif
  block_til_complete = .true.
  if (present(complete)) block_til_complete = complete

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_update_domains(array, mom_dom%mpp_domain, flags=dirflag, &
                        complete=block_til_complete, position=position, &
                        whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_update_domains(array, mom_dom%mpp_domain, flags=dirflag, &
                        complete=block_til_complete, position=position)
  endif

end subroutine pass_var_2d

function pass_var_start_2d(array, MOM_dom, sideflag, position, complete, halo)
  real, dimension(:,:),   intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                                    !! halo updates should be completed before
                                                    !! progress resumes.  Omitting complete is the
                                                    !! same as setting complete to .true.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
  integer                               :: pass_var_start_2d  !<The integer index for this update.
! Arguments: array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in)      position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be initiated immediately or wait for second
!                       pass_..._start call.  Omitting complete is the same as
!                       setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    pass_var_start_2d = mpp_start_update_domains(array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position, &
                            whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    pass_var_start_2d = mpp_start_update_domains(array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position)
  endif
end function pass_var_start_2d

function pass_var_start_3d(array, MOM_dom, sideflag, position, complete, halo)
  real, dimension(:,:,:), intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                                    !! halo updates should be completed before
                                                    !! progress resumes.  Omitting complete is the
                                                    !! same as setting complete to .true.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
  integer                               :: pass_var_start_3d  !< The integer index for this update.
! Arguments: array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in)      position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be initiated immediately or wait for second
!                       pass_..._start call.  Omitting complete is the same as
!                       setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    pass_var_start_3d = mpp_start_update_domains(array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position, &
                            whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    pass_var_start_3d = mpp_start_update_domains(array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position)
  endif
end function pass_var_start_3d

! #@# This subroutine needs a doxygen description
subroutine pass_var_complete_2d(id_update, array, MOM_dom, sideflag, position, halo)
  integer,                intent(in)    :: id_update !< The integer id of this update which has
                                                    !! been returned from a previous call to
                                                    !! pass_var_start.
  real, dimension(:,:),   intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: id_update - The integer id of this update which has been returned
!                        from a previous call to pass_var_start.
!  (inout)   array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in)      position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_complete_update_domains(id_update, array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position, &
                            whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_complete_update_domains(id_update, array, mom_dom%mpp_domain, &
                                     flags=dirflag, position=position)
  endif

end subroutine pass_var_complete_2d

! #@# This subroutine needs a doxygen description
subroutine pass_var_complete_3d(id_update, array, MOM_dom, sideflag, position, halo)
  integer,                intent(in)    :: id_update !< The integer id of this update which has
                                                    !! been returned from a previous call to
                                                    !! pass_var_start.
  real, dimension(:,:,:), intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: id_update - The integer id of this update which has been returned
!                        from a previous call to pass_var_start.
!  (inout)   array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in)      position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_complete_update_domains(id_update, array, mom_dom%mpp_domain, &
                            flags=dirflag, position=position, &
                            whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_complete_update_domains(id_update, array, mom_dom%mpp_domain, &
                                     flags=dirflag, position=position)
  endif

end subroutine pass_var_complete_3d

! #@# This subroutine needs a doxygen description
subroutine pass_vector_2d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo)
  real, dimension(:,:),  intent(inout) :: u_cmpt    !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:),  intent(inout) :: v_cmpt    !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type), intent(inout) :: MOM_dom   !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,     optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,     optional, intent(in)    :: stagger   !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  logical,     optional, intent(in)    :: complete  !< An optional argument indicating whether the
                                     !! halo updates should be completed before progress resumes.
                                     !! Omitting complete is the same as setting complete to .true.
  integer,     optional, intent(in)    :: halo      !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be completed before progress resumes.  Omitting
!                       complete is the same as setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.

  integer :: stagger_local
  integer :: dirflag
  logical :: block_til_complete

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif
  block_til_complete = .true.
  if (present(complete)) block_til_complete = complete

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_update_domains(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                   gridtype=stagger_local, complete = block_til_complete, &
                   whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_update_domains(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                   gridtype=stagger_local, complete = block_til_complete)
  endif

end subroutine pass_vector_2d

! #@# This subroutine needs a doxygen description
subroutine fill_vector_symmetric_edges_2d(u_cmpt, v_cmpt, MOM_dom, stagger, scalar)
  real, dimension(:,:),  intent(inout) :: u_cmpt  !< The nominal zonal (u) component of the vector
                                                  !! pair which is having its halos points
                                                  !! exchanged.
  real, dimension(:,:),  intent(inout) :: v_cmpt  !< The nominal meridional (v) component of the
                                                  !! vector pair which is having its halos points
                                                  !! exchanged.
  type(mom_domain_type), intent(inout) :: MOM_dom !< The MOM_domain_type containing the mpp_domain
                                                  !! needed to determine where data should be
                                                  !! sent.
  integer,     optional, intent(in)    :: stagger !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  logical,     optional, intent(in)    :: scalar !< An optional argument indicating whether.
! Arguments: u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in)      scalar -  An optional argument indicating whether

  integer :: stagger_local
  integer :: dirflag
  integer :: i, j, isc, iec, jsc, jec, isd, ied, jsd, jed, IscB, IecB, JscB, JecB
  real, allocatable, dimension(:) :: sbuff_x, sbuff_y, wbuff_x, wbuff_y
  logical :: block_til_complete

  if (.not. mom_dom%symmetric) return

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  if (.not.(stagger_local == cgrid_ne .or. stagger_local == bgrid_ne)) return

  call mpp_get_compute_domain(mom_dom%mpp_domain, isc, iec, jsc, jec)
  call mpp_get_data_domain(mom_dom%mpp_domain, isd, ied, jsd, jed)

  ! Adjust isc, etc., to account for the fact that the input arrays indices all
  ! start at 1 (and are effectively on a SW grid!).
  isc = isc - (isd-1) ; iec = iec - (isd-1)
  jsc = jsc - (jsd-1) ; jec = jec - (jsd-1)
  iscb = isc ; iecb = iec+1 ; jscb = jsc ; jecb = jec+1

  dirflag = to_all ! 60
  if (present(scalar)) then ; if (scalar) dirflag = to_all+scalar_pair ; endif

  if (stagger_local == cgrid_ne) then
    allocate(wbuff_x(jsc:jec)) ; allocate(sbuff_y(isc:iec))
    wbuff_x(:) = 0.0 ; sbuff_y(:) = 0.0
    call mpp_get_boundary(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                          wbufferx=wbuff_x, sbuffery=sbuff_y, &
                          gridtype=cgrid_ne)
    do i=isc,iec
      v_cmpt(i,jscb) = sbuff_y(i)
    enddo
    do j=jsc,jec
      u_cmpt(iscb,j) = wbuff_x(j)
    enddo
    deallocate(wbuff_x) ; deallocate(sbuff_y)
  elseif  (stagger_local == bgrid_ne) then
    allocate(wbuff_x(jscb:jecb)) ; allocate(sbuff_x(iscb:iecb))
    allocate(wbuff_y(jscb:jecb)) ; allocate(sbuff_y(iscb:iecb))
    wbuff_x(:) = 0.0 ; wbuff_y(:) = 0.0 ; sbuff_x(:) = 0.0 ; sbuff_y(:) = 0.0
    call mpp_get_boundary(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                          wbufferx=wbuff_x, sbufferx=sbuff_x, &
                          wbuffery=wbuff_y, sbuffery=sbuff_y, &
                          gridtype=bgrid_ne)
    do i=iscb,iecb
      u_cmpt(i,jscb) = sbuff_x(i) ; v_cmpt(i,jscb) = sbuff_y(i)
    enddo
    do j=jscb,jecb
      u_cmpt(iscb,j) = wbuff_x(j) ; v_cmpt(iscb,j) = wbuff_y(j)
    enddo
    deallocate(wbuff_x) ; deallocate(sbuff_x)
    deallocate(wbuff_y) ; deallocate(sbuff_y)
  endif

end subroutine fill_vector_symmetric_edges_2d

! #@# This subroutine needs a doxygen description
subroutine pass_vector_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo)
  real, dimension(:,:,:), intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:,:), intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                     !! halo updates should be completed before progress resumes.
                                     !! Omitting complete is the same as setting complete to .true.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be completed before progress resumes.  Omitting
!                       complete is the same as setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.

  integer :: stagger_local
  integer :: dirflag
  logical :: block_til_complete

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif
  block_til_complete = .true.
  if (present(complete)) block_til_complete = complete

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_update_domains(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                   gridtype=stagger_local, complete = block_til_complete, &
                   whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_update_domains(u_cmpt, v_cmpt, mom_dom%mpp_domain, flags=dirflag, &
                   gridtype=stagger_local, complete = block_til_complete)
  endif

end subroutine pass_vector_3d

function pass_vector_start_2d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo)
  real, dimension(:,:),   intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:),   intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                     !! halo updates should be completed before progress resumes.
                                     !! Omitting complete is the same as setting complete to .true.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
  integer                               :: pass_vector_start_2d !< The integer index for this
                                                                !! update.
! Arguments: u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be initiated immediately or wait for second
!                       pass_..._start call.  Omitting complete is the same as
!                       setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    pass_vector_start_2d = mpp_start_update_domains(u_cmpt, v_cmpt, &
        mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local, &
        whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    pass_vector_start_2d = mpp_start_update_domains(u_cmpt, v_cmpt, &
        mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local)
  endif

end function pass_vector_start_2d

function pass_vector_start_3d(u_cmpt, v_cmpt, MOM_dom, direction, stagger, complete, halo)
  real, dimension(:,:,:), intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:,:), intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  logical,      optional, intent(in)    :: complete !< An optional argument indicating whether the
                                     !! halo updates should be completed before progress resumes.
                                     !! Omitting complete is the same as setting complete to .true.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
  integer                               :: pass_vector_start_3d !< The integer index for this
                                                                !! update.
! Arguments: u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in)      complete - An optional argument indicating whether the halo updates
!                       should be initiated immediately or wait for second
!                       pass_..._start call.  Omitting complete is the same as
!                       setting complete to .true.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    pass_vector_start_3d = mpp_start_update_domains(u_cmpt, v_cmpt, &
        mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local, &
        whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    pass_vector_start_3d = mpp_start_update_domains(u_cmpt, v_cmpt, &
        mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local)
  endif

end function pass_vector_start_3d

! #@# This subroutine needs a doxygen description
subroutine pass_vector_complete_2d(id_update, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo)
  integer,                intent(in)    :: id_update !< The integer id of this update which has been
                                                    !! returned from a previous call to
                                                    !! pass_var_start.
  real, dimension(:,:),   intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:),   intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: id_update - The integer id of this update which has been returned
!                        from a previous call to pass_var_start.
!  (inout)   u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_complete_update_domains(id_update, u_cmpt, v_cmpt, &
             mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local, &
             whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_complete_update_domains(id_update, u_cmpt, v_cmpt, &
             mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local)
  endif

end subroutine pass_vector_complete_2d

! #@# This subroutine needs a doxygen description
subroutine pass_vector_complete_3d(id_update, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo)
  integer,                intent(in)    :: id_update !< The integer id of this update which has been
                                                    !! returned from a previous call to
                                                    !! pass_var_start.
  real, dimension(:,:,:), intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:,:), intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments: id_update - The integer id of this update which has been returned
!                        from a previous call to pass_var_start.
!  (inout)   u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
!  (return value) - The integer index for this update.
  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_complete_update_domains(id_update, u_cmpt, v_cmpt, &
             mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local, &
                   whalo=halo, ehalo=halo, shalo=halo, nhalo=halo)
  else
    call mpp_complete_update_domains(id_update, u_cmpt, v_cmpt, &
             mom_dom%mpp_domain, flags=dirflag, gridtype=stagger_local)
  endif

end subroutine pass_vector_complete_3d

! #@# This subroutine needs a doxygen description
subroutine create_var_group_pass_2d(group, array, MOM_dom, sideflag, position, &
                                    halo)
  type(group_pass_type),  intent(inout) :: group    !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  real, dimension(:,:),   intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments:
!  (inout)   group - The data type that store information for group update.
!                    This data will be used in do_group_pass.
!  (inout)   array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in,opt)  sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in,opt)  position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (mpp_group_update_initialized(group)) then
    call mpp_reset_group_update_field(group,array)
  elseif (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_create_group_update(group, array, mom_dom%mpp_domain, flags=dirflag, &
                                 position=position, whalo=halo, ehalo=halo, &
                                 shalo=halo, nhalo=halo)
  else
    call mpp_create_group_update(group, array, mom_dom%mpp_domain, flags=dirflag, &
                                 position=position)
  endif

end subroutine create_var_group_pass_2d

! #@# This subroutine needs a doxygen description
subroutine create_var_group_pass_3d(group, array, MOM_dom, sideflag, position, halo)
  type(group_pass_type),  intent(inout) :: group    !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  real, dimension(:,:,:), intent(inout) :: array    !< The array which is having its halos points
                                                    !! exchanged.
  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: sideflag !< An optional integer indicating which
      !! directions the data should be sent. It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH.  For example, TO_EAST sends the data to the processor to the east,
      !! so the halos on the western side are filled.  TO_ALL is the default if sideflag is omitted.
  integer,      optional, intent(in)    :: position !< An optional argument indicating the position.
                                                    !! This is usally CORNER, but is CENTER
                                                    !! by default.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments:
!  (inout)   group - The data type that store information for group update.
!                    This data will be used in do_group_pass.
!  (inout)   array - The array which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in,opt)  sideflag - An optional integer indicating which directions the
!                       data should be sent.  It is TO_ALL or the sum of any of
!                       TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH.  For example,
!                       TO_EAST sends the data to the processor to the east, so
!                       the halos on the western side are filled.  TO_ALL is
!                       the default if sideflag is omitted.
!  (in,opt)  position - An optional argument indicating the position.  This is
!                       may be CORNER, but is CENTER by default.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: dirflag

  dirflag = to_all ! 60
  if (present(sideflag)) then ; if (sideflag > 0) dirflag = sideflag ; endif

  if (mpp_group_update_initialized(group)) then
    call mpp_reset_group_update_field(group,array)
  elseif (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_create_group_update(group, array, mom_dom%mpp_domain, flags=dirflag, &
                                 position=position, whalo=halo, ehalo=halo, &
                                 shalo=halo, nhalo=halo)
  else
    call mpp_create_group_update(group, array, mom_dom%mpp_domain, flags=dirflag, &
                                 position=position)
  endif

end subroutine create_var_group_pass_3d

! #@# This subroutine needs a doxygen description
subroutine create_vector_group_pass_2d(group, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo)
  type(group_pass_type),  intent(inout) :: group    !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  real, dimension(:,:),   intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:),   intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.

  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments:
!  (inout)   group - The data type that store information for group update.
!                    This data will be used in do_group_pass.
!  (inout)   u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.
  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (mpp_group_update_initialized(group)) then
    call mpp_reset_group_update_field(group,u_cmpt, v_cmpt)
  elseif (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_create_group_update(group, u_cmpt, v_cmpt, mom_dom%mpp_domain, &
            flags=dirflag, gridtype=stagger_local, whalo=halo, ehalo=halo, &
            shalo=halo, nhalo=halo)
  else
    call mpp_create_group_update(group, u_cmpt, v_cmpt, mom_dom%mpp_domain, &
            flags=dirflag, gridtype=stagger_local)
  endif

end subroutine create_vector_group_pass_2d

! #@# This subroutine needs a doxygen description
subroutine create_vector_group_pass_3d(group, u_cmpt, v_cmpt, MOM_dom, direction, stagger, halo)
  type(group_pass_type),  intent(inout) :: group    !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  real, dimension(:,:,:), intent(inout) :: u_cmpt   !< The nominal zonal (u) component of the vector
                                                    !! pair which is having its halos points
                                                    !! exchanged.
  real, dimension(:,:,:), intent(inout) :: v_cmpt   !< The nominal meridional (v) component of the
                                                    !! vector pair which is having its halos points
                                                    !! exchanged.

  type(mom_domain_type),  intent(inout) :: MOM_dom  !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  integer,      optional, intent(in)    :: direction !< An optional integer indicating which
      !! directions the data should be sent.  It is TO_ALL or the sum of any of TO_EAST, TO_WEST,
      !! TO_NORTH, and TO_SOUTH, possibly plus SCALAR_PAIR if these are paired non-directional
      !! scalars discretized at the typical vector component locations.  For example, TO_EAST sends
      !! the data to the processor to the east, so the halos on the western side are filled. TO_ALL
      !! is the default if omitted.
  integer,      optional, intent(in)    :: stagger  !< An optional flag, which may be one of A_GRID,
                     !! BGRID_NE, or CGRID_NE, indicating where the two components of the vector are
                     !! discretized. Omitting stagger is the same as setting it to CGRID_NE.
  integer,      optional, intent(in)    :: halo     !< The size of the halo to update - the full
                                                    !! halo by default.
! Arguments:
!  (inout)   group - The data type that store information for group update.
!                    This data will be used in do_group_pass.
!  (inout)   u_cmpt - The nominal zonal (u) component of the vector pair which
!                     is having its halos points exchanged.
!  (inout)   v_cmpt - The nominal meridional (v) component of the vector pair
!                     which is having its halos points exchanged.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.
!  (in)      direction - An optional integer indicating which directions the
!                        data should be sent.  It is TO_ALL or the sum of any of
!                        TO_EAST, TO_WEST, TO_NORTH, and TO_SOUTH, possibly
!                        plus SCALAR_PAIR if these are paired non-directional
!                        scalars discretized at the typical vector component
!                        locations.  For example, TO_EAST sends the data to the
!                        processor to the east, so the halos on the western
!                        side are filled.  TO_ALL is the default if omitted.
!  (in)      stagger - An optional flag, which may be one of A_GRID, BGRID_NE,
!                      or CGRID_NE, indicating where the two components of the
!                      vector are discretized.  Omitting stagger is the same as
!                      setting it to CGRID_NE.
!  (in,opt)  halo - The size of the halo to update - the full halo by default.

  integer :: stagger_local
  integer :: dirflag

  stagger_local = cgrid_ne ! Default value for type of grid
  if (present(stagger)) stagger_local = stagger

  dirflag = to_all ! 60
  if (present(direction)) then ; if (direction > 0) dirflag = direction ; endif

  if (mpp_group_update_initialized(group)) then
    call mpp_reset_group_update_field(group,u_cmpt, v_cmpt)
  elseif (present(halo) .and. mom_dom%thin_halo_updates) then
    call mpp_create_group_update(group, u_cmpt, v_cmpt, mom_dom%mpp_domain, &
            flags=dirflag, gridtype=stagger_local, whalo=halo, ehalo=halo, &
            shalo=halo, nhalo=halo)
  else
    call mpp_create_group_update(group, u_cmpt, v_cmpt, mom_dom%mpp_domain, &
            flags=dirflag, gridtype=stagger_local)
  endif

end subroutine create_vector_group_pass_3d

! #@# This subroutine needs a doxygen description
subroutine do_group_pass(group, MOM_dom)
  type(group_pass_type), intent(inout) :: group     !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  type(mom_domain_type), intent(inout) :: MOM_dom   !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  real                                 :: d_type

! Arguments:
!  (inout)   group - The data type that store information for group update.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.

  call mpp_do_group_update(group, mom_dom%mpp_domain, d_type)

end subroutine do_group_pass

! #@# This subroutine needs a doxygen description
subroutine start_group_pass(group, MOM_dom)
  type(group_pass_type), intent(inout) :: group     !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  type(mom_domain_type), intent(inout) :: MOM_dom   !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  real                                 :: d_type

! Arguments:
!  (inout)   group - The data type that store information for group update.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.

  call mpp_start_group_update(group, mom_dom%mpp_domain, d_type)

end subroutine start_group_pass

! #@# This subroutine needs a doxygen description
subroutine complete_group_pass(group, MOM_dom)
  type(group_pass_type), intent(inout) :: group     !< The data type that store information for
                                                    !! group update. This data will be used in
                                                    !! do_group_pass.
  type(mom_domain_type), intent(inout) :: MOM_dom   !< The MOM_domain_type containing the mpp_domain
                                                    !! needed to determine where data should be
                                                    !! sent.
  real                                 :: d_type

! Arguments:
!  (inout)   group - The data type that store information for group update.
!  (in)      MOM_dom - The MOM_domain_type containing the mpp_domain needed to
!                      determine where data should be sent.

  call mpp_complete_group_update(group, mom_dom%mpp_domain, d_type)

end subroutine complete_group_pass

! #@# This subroutine needs a doxygen description.
subroutine mom_domains_init(MOM_dom, param_file, symmetric, static_memory, &
                            NIHALO, NJHALO, NIGLOBAL, NJGLOBAL, NIPROC, NJPROC, &
                            min_halo, domain_name, include_name, param_suffix)
  type(mom_domain_type),           pointer       :: MOM_dom      !< A pointer to the MOM_domain_type
                                                                 !! being defined here.
  type(param_file_type),           intent(in)    :: param_file   !< A structure to parse for
                                                                 !! run-time parameters
  logical, optional,               intent(in)    :: symmetric    !< If present, this specifies
                                            !! whether this domain is symmetric, regardless of
                                            !! whether the macro SYMMETRIC_MEMORY_ is defined.
  logical, optional,               intent(in)    :: static_memory !< If present and true, this
                         !! domain type is set up for static memory and error checking of
                         !! various input values is performed against those in the input file.
  integer, optional,               intent(in)    :: NIHALO       !< Default halo sizes, required
                                                                 !! with static memory.
  integer, optional,               intent(in)    :: NJHALO       !< Default halo sizes, required
                                                                 !! with static memory.
  integer, optional,               intent(in)    :: NIGLOBAL     !< Total domain sizes, required
                                                                 !! with static memory.
  integer, optional,               intent(in)    :: NJGLOBAL     !< Total domain sizes, required
                                                                 !! with static memory.
  integer, optional,               intent(in)    :: NIPROC       !< Processor counts, required with
                                                                 !! static memory.
  integer, optional,               intent(in)    :: NJPROC       !< Processor counts, required with
                                                                 !! static memory.
  integer, dimension(2), optional, intent(inout) :: min_halo     !< If present, this sets the
                                        !! minimum halo size for this domain in the x- and y-
                                        !! directions, and returns the actual halo size used.
  character(len=*),      optional, intent(in)    :: domain_name  !< A name for this domain, "MOM"
                                                                 !! if missing.
  character(len=*),      optional, intent(in)    :: include_name !< A name for model's include file,
                                                                 !! "MOM_memory.h" if missing.
  character(len=*),      optional, intent(in)    :: param_suffix !< A suffix to apply to
                                                                 !! layout-specific parameters.


! Arguments: MOM_dom - A pointer to the MOM_domain_type being defined here.
!  (in)      param_file - A structure indicating the open file to parse for
!                         model parameter values.
!  (in,opt)  symmetric - If present, this specifies whether this domain
!                        is symmetric, regardless of whether the macro
!                        SYMMETRIC_MEMORY_ is defined.
!  (in,opt)  static_memory - If present and true, this domain type is set up for
!                            static memory and error checking of various input
!                            values is performed against those in the input file.
!  (in,opt)  NIHALO, NJHALO - Default halo sizes, required with static memory.
!  (in,opt)  NIGLOBAL, NJGLOBAL - Total domain sizes, required with static memory.
!  (in,opt)  NIPROC, NJPROC - Processor counts, required with static memory.
!  (in,opt)  min_halo - If present, this sets the minimum halo size for this
!                       domain in the x- and y- directions, and returns the
!                       actual halo size used.
!  (in,opt)  domain_name - A name for this domain, "MOM" if missing.
!  (in,opt)  include_name - A name for model's include file, "MOM_memory.h" if missing.
!  (in,opt)  param_suffix - A suffix to apply to layout-specific parameters.

  integer, dimension(2) :: layout = (/ 1, 1 /)
  integer, dimension(2) :: io_layout = (/ 0, 0 /)
  integer, dimension(4) :: global_indices
!$ integer :: ocean_nthreads       ! Number of Openmp threads
!$ integer :: get_cpu_affinity, omp_get_thread_num, omp_get_num_threads
!$ integer :: omp_cores_per_node, adder, base_cpu
!$ logical :: ocean_omp_hyper_thread
  integer :: nihalo_dflt, njhalo_dflt
  integer :: pe, proc_used
  integer :: X_FLAGS, Y_FLAGS
  logical :: reentrant_x, reentrant_y, tripolar_N, is_static
  logical            :: mask_table_exists
  character(len=128) :: mask_table, inputdir
  character(len=64)  :: dom_name, inc_nm
  character(len=200) :: mesg

  integer :: xsiz, ysiz, nip_parsed, njp_parsed
  integer :: isc,iec,jsc,jec ! The bounding indices of the computational domain.
  character(len=8) :: char_xsiz, char_ysiz, char_niglobal, char_njglobal
  character(len=40) :: nihalo_nm, njhalo_nm, layout_nm, io_layout_nm, masktable_nm
  character(len=40) :: niproc_nm, njproc_nm

! This include declares and sets the variable "version".
#include "version_variable.h"
  character(len=40)  :: mdl ! This module's name.

  if (.not.associated(mom_dom)) then
    allocate(mom_dom)
    allocate(mom_dom%mpp_domain)
  endif

  pe = pe_here()
  proc_used = num_pes()

  mdl = "MOM_domains"

  mom_dom%symmetric = .true.
  if (present(symmetric)) then ; mom_dom%symmetric = symmetric ; endif
  if (present(min_halo)) mdl = trim(mdl)//" min_halo"

  dom_name = "MOM" ; inc_nm = "MOM_memory.h"
  if (present(domain_name)) dom_name = trim(domain_name)
  if (present(include_name)) inc_nm = trim(include_name)

  nihalo_nm = "NIHALO" ; njhalo_nm = "NJHALO"
  layout_nm = "LAYOUT" ; io_layout_nm = "IO_LAYOUT" ; masktable_nm = "MASKTABLE"
  niproc_nm = "NIPROC" ; njproc_nm = "NJPROC"
  if (present(param_suffix)) then ; if (len(trim(adjustl(param_suffix))) > 0) then
    nihalo_nm = "NIHALO"//(trim(adjustl(param_suffix)))
    njhalo_nm = "NJHALO"//(trim(adjustl(param_suffix)))
    layout_nm = "LAYOUT"//(trim(adjustl(param_suffix)))
    io_layout_nm = "IO_LAYOUT"//(trim(adjustl(param_suffix)))
    masktable_nm = "MASKTABLE"//(trim(adjustl(param_suffix)))
    niproc_nm = "NIPROC"//(trim(adjustl(param_suffix)))
    njproc_nm = "NJPROC"//(trim(adjustl(param_suffix)))
  endif ; endif

  is_static = .false. ; if (present(static_memory)) is_static = static_memory
  if (is_static) then
    if (.not.present(nihalo)) call mom_error(fatal, "NIHALO must be "// &
      "present in the call to MOM_domains_init with static memory.")
    if (.not.present(njhalo)) call mom_error(fatal, "NJHALO must be "// &
      "present in the call to MOM_domains_init with static memory.")
    if (.not.present(niglobal)) call mom_error(fatal, "NIGLOBAL must be "// &
      "present in the call to MOM_domains_init with static memory.")
    if (.not.present(njglobal)) call mom_error(fatal, "NJGLOBAL must be "// &
      "present in the call to MOM_domains_init with static memory.")
    if (.not.present(niproc)) call mom_error(fatal, "NIPROC must be "// &
      "present in the call to MOM_domains_init with static memory.")
    if (.not.present(njproc)) call mom_error(fatal, "NJPROC must be "// &
      "present in the call to MOM_domains_init with static memory.")
  endif

  ! Read all relevant parameters and write them to the model log.
  call log_version(param_file, mdl, version, "")
  call get_param(param_file, mdl, "REENTRANT_X", reentrant_x, &
                 "If true, the domain is zonally reentrant.", default=.true.)
  call get_param(param_file, mdl, "REENTRANT_Y", reentrant_y, &
                 "If true, the domain is meridionally reentrant.", &
                 default=.false.)
  call get_param(param_file, mdl, "TRIPOLAR_N", tripolar_n, &
                 "Use tripolar connectivity at the northern edge of the \n"//&
                 "domain.  With TRIPOLAR_N, NIGLOBAL must be even.", &
                 default=.false.)

#ifndef NOT_SET_AFFINITY
!$ call get_param(param_file, mdl, "OCEAN_OMP_THREADS", ocean_nthreads, &
!$            "The number of OpenMP threads that MOM6 will use.", &
!$            default = 1, layoutParam=.true.)
!$ call get_param(param_file, mdl, "OCEAN_OMP_HYPER_THREAD", ocean_omp_hyper_thread, &
!$            "If True, use hyper-threading.", default = .false., layoutParam=.true.)
!$ if (ocean_omp_hyper_thread) then
!$   call get_param(param_file, mdl, "OMP_CORES_PER_NODE", omp_cores_per_node, &
!$            "Number of cores per node needed for hyper-threading.", &
!$            fail_if_missing=.true., layoutParam=.true.)
!$ endif
!$ call omp_set_num_threads(ocean_nthreads)
!$OMP PARALLEL private(adder)
!$ base_cpu = get_cpu_affinity()
!$ if (ocean_omp_hyper_thread) then
!$   if (mod(omp_get_thread_num(),2) == 0) then
!$     adder = omp_get_thread_num()/2
!$   else
!$     adder = omp_cores_per_node + omp_get_thread_num()/2
!$   endif
!$ else
!$   adder = omp_get_thread_num()
!$ endif
!$ call set_cpu_affinity(base_cpu + adder)
!!$     write(6,*) " ocean  ", omp_get_num_threads(), get_cpu_affinity(), adder, omp_get_thread_num()
!!$     call flush(6)
!$OMP END PARALLEL
#endif

  call log_param(param_file, mdl, "!SYMMETRIC_MEMORY_", mom_dom%symmetric, &
                 "If defined, the velocity point data domain includes \n"//&
                 "every face of the thickness points. In other words, \n"//&
                 "some arrays are larger than others, depending on where \n"//&
                 "they are on the staggered grid.  Also, the starting \n"//&
                 "index of the velocity-point arrays is usually 0, not 1. \n"//&
                 "This can only be set at compile time.",&
                 layoutparam=.true.)
  call get_param(param_file, mdl, "NONBLOCKING_UPDATES", mom_dom%nonblocking_updates, &
                 "If true, non-blocking halo updates may be used.", &
                 default=.false., layoutparam=.true.)
  call get_param(param_file, mdl, "THIN_HALO_UPDATES", mom_dom%thin_halo_updates, &
                 "If true, optional arguments may be used to specify the \n"//&
                 "The width of the halos that are updated with each call.", &
                 default=.true., layoutparam=.true.)

  nihalo_dflt = 4 ; njhalo_dflt = 4
  if (present(nihalo)) nihalo_dflt = nihalo
  if (present(njhalo)) njhalo_dflt = njhalo

  call log_param(param_file, mdl, "!STATIC_MEMORY_", is_static, &
                 "If STATIC_MEMORY_ is defined, the principle variables \n"//&
                 "will have sizes that are statically determined at \n"//&
                 "compile time.  Otherwise the sizes are not determined \n"//&
                 "until run time. The STATIC option is substantially \n"//&
                 "faster, but does not allow the PE count to be changed \n"//&
                 "at run time.  This can only be set at compile time.",&
                 layoutparam=.true.)

  call get_param(param_file, mdl, trim(nihalo_nm), mom_dom%nihalo, &
                 "The number of halo points on each side in the \n"//&
                 "x-direction.  With STATIC_MEMORY_ this is set as NIHALO_ \n"//&
                 "in "//trim(inc_nm)//" at compile time; without STATIC_MEMORY_ \n"//&
                 "the default is NIHALO_ in "//trim(inc_nm)//" (if defined) or 2.", &
                 default=4, static_value=nihalo_dflt, layoutparam=.true.)
  call get_param(param_file, mdl, trim(njhalo_nm), mom_dom%njhalo, &
                 "The number of halo points on each side in the \n"//&
                 "y-direction.  With STATIC_MEMORY_ this is set as NJHALO_ \n"//&
                 "in "//trim(inc_nm)//" at compile time; without STATIC_MEMORY_ \n"//&
                 "the default is NJHALO_ in "//trim(inc_nm)//" (if defined) or 2.", &
                 default=4, static_value=njhalo_dflt, layoutparam=.true.)
  if (present(min_halo)) then
    mom_dom%nihalo = max(mom_dom%nihalo, min_halo(1))
    min_halo(1) = mom_dom%nihalo
    mom_dom%njhalo = max(mom_dom%njhalo, min_halo(2))
    min_halo(2) = mom_dom%njhalo
    call log_param(param_file, mdl, "!NIHALO min_halo", mom_dom%nihalo, layoutparam=.true.)
    call log_param(param_file, mdl, "!NJHALO min_halo", mom_dom%nihalo, layoutparam=.true.)
  endif
  if (is_static) then
    call get_param(param_file, mdl, "NIGLOBAL", mom_dom%niglobal, &
                 "The total number of thickness grid points in the \n"//&
                 "x-direction in the physical domain. With STATIC_MEMORY_ \n"//&
                 "this is set in "//trim(inc_nm)//" at compile time.", &
                 static_value=niglobal)
    call get_param(param_file, mdl, "NJGLOBAL", mom_dom%njglobal, &
                 "The total number of thickness grid points in the \n"//&
                 "y-direction in the physical domain. With STATIC_MEMORY_ \n"//&
                 "this is set in "//trim(inc_nm)//" at compile time.", &
                 static_value=njglobal)
    if (mom_dom%niglobal /= niglobal) call mom_error(fatal,"MOM_domains_init: " // &
     "static mismatch for NIGLOBAL_ domain size. Header file does not match input namelist")
    if (mom_dom%njglobal /= njglobal) call mom_error(fatal,"MOM_domains_init: " // &
     "static mismatch for NJGLOBAL_ domain size. Header file does not match input namelist")

    if (.not.present(min_halo)) then
      if (mom_dom%nihalo /= nihalo) call mom_error(fatal,"MOM_domains_init: " // &
             "static mismatch for "//trim(nihalo_nm)//" domain size")
      if (mom_dom%njhalo /= njhalo) call mom_error(fatal,"MOM_domains_init: " // &
             "static mismatch for "//trim(njhalo_nm)//" domain size")
    endif
  else
    call get_param(param_file, mdl, "NIGLOBAL", mom_dom%niglobal, &
                 "The total number of thickness grid points in the \n"//&
                 "x-direction in the physical domain. With STATIC_MEMORY_ \n"//&
                 "this is set in "//trim(inc_nm)//" at compile time.", &
                 fail_if_missing=.true.)
    call get_param(param_file, mdl, "NJGLOBAL", mom_dom%njglobal, &
                 "The total number of thickness grid points in the \n"//&
                 "y-direction in the physical domain. With STATIC_MEMORY_ \n"//&
                 "this is set in "//trim(inc_nm)//" at compile time.", &
                 fail_if_missing=.true.)
  endif

  global_indices(1) = 1 ; global_indices(2) = mom_dom%niglobal
  global_indices(3) = 1 ; global_indices(4) = mom_dom%njglobal

  call get_param(param_file, mdl, "INPUTDIR", inputdir, do_not_log=.true., default=".")
  inputdir = slasher(inputdir)

  call get_param(param_file, mdl, trim(masktable_nm), mask_table, &
                 "A text file to specify n_mask, layout and mask_list. \n"//&
                 "This feature masks out processors that contain only land points. \n"//&
                 "The first line of mask_table is the number of regions to be masked out.\n"//&
                 "The second line is the layout of the model and must be \n"//&
                 "consistent with the actual model layout.\n"//&
                 "The following (n_mask) lines give the logical positions \n"//&
                 "of the processors that are masked out. The mask_table \n"//&
                 "can be created by tools like check_mask. The \n"//&
                 "following example of mask_table masks out 2 processors, \n"//&
                 "(1,2) and (3,6), out of the 24 in a 4x6 layout: \n"//&
                 " 2\n 4,6\n 1,2\n 3,6\n", default="MOM_mask_table", &
                 layoutparam=.true.)
  mask_table = trim(inputdir)//trim(mask_table)
  mask_table_exists = file_exist(mask_table)

  if (is_static) then
    layout(1) = niproc ; layout(2) = njproc
  else
    call get_param(param_file, mdl, trim(layout_nm), layout, &
                 "The processor layout to be used, or 0, 0 to automatically \n"//&
                 "set the layout based on the number of processors.", default=0, &
                 do_not_log=.true.)
    call get_param(param_file, mdl, trim(niproc_nm), nip_parsed, &
                 "The number of processors in the x-direction.", default=-1, &
                 do_not_log=.true.)
    call get_param(param_file, mdl, trim(njproc_nm), njp_parsed, &
                 "The number of processors in the y-direction.", default=-1, &
                 do_not_log=.true.)
    if (nip_parsed > -1) then
      if ((layout(1) > 0) .and. (layout(1) /= nip_parsed)) &
        call mom_error(fatal, trim(layout_nm)//" and "//trim(niproc_nm)//" set inconsistently. "//&
                              "Only LAYOUT should be used.")
      layout(1) = nip_parsed
      call mom_mesg(trim(niproc_nm)//" used to set "//trim(layout_nm)//" in dynamic mode.  "//&
                    "Shift to using "//trim(layout_nm)//" instead.")
    endif
    if (njp_parsed > -1) then
      if ((layout(2) > 0) .and. (layout(2) /= njp_parsed)) &
        call mom_error(fatal, trim(layout_nm)//" and "//trim(njproc_nm)//" set inconsistently. "//&
                              "Only "//trim(layout_nm)//" should be used.")
      layout(2) = njp_parsed
      call mom_mesg(trim(njproc_nm)//" used to set "//trim(layout_nm)//" in dynamic mode.  "//&
                    "Shift to using "//trim(layout_nm)//" instead.")
    endif

    if ( layout(1)==0 .and. layout(2)==0 ) &
      call mpp_define_layout(global_indices, proc_used, layout)
    if ( layout(1)/=0 .and. layout(2)==0 ) layout(2) = proc_used/layout(1)
    if ( layout(1)==0 .and. layout(2)/=0 ) layout(1) = proc_used/layout(2)

    if (layout(1)*layout(2) /= proc_used .and. (.not. mask_table_exists) ) then
      write(mesg,'("MOM_domains_init: The product of the two components of layout, ", &
            &      2i4,", is not the number of PEs used, ",i5,".")') &
            layout(1),layout(2),proc_used
      call mom_error(fatal, mesg)
    endif
  endif
  call log_param(param_file, mdl, trim(niproc_nm), layout(1), &
                 "The number of processors in the x-direction. With \n"//&
                 "STATIC_MEMORY_ this is set in "//trim(inc_nm)//" at compile time.",&
                 layoutparam=.true.)
  call log_param(param_file, mdl, trim(njproc_nm), layout(2), &
                 "The number of processors in the x-direction. With \n"//&
                 "STATIC_MEMORY_ this is set in "//trim(inc_nm)//" at compile time.",&
                 layoutparam=.true.)
  call log_param(param_file, mdl, trim(layout_nm), layout, &
                 "The processor layout that was acutally used.",&
                 layoutparam=.true.)

  ! Idiot check that fewer PEs than columns have been requested
  if (layout(1)*layout(2)>mom_dom%niglobal*mom_dom%njglobal)  then
    write(mesg,'(a,2(i5,x,a))') 'You requested to use',layout(1)*layout(2), &
      'PEs but there are only',mom_dom%niglobal*mom_dom%njglobal,'columns in the model'
    call mom_error(fatal, mesg)
  endif

  if (mask_table_exists) then
    call mom_error(note, 'MOM_domains_init: reading maskmap information from '//&
                         trim(mask_table))
    allocate(mom_dom%maskmap(layout(1), layout(2)))
    call parse_mask_table(mask_table, mom_dom%maskmap, dom_name)
  endif

  !   Set up the I/O lay-out, and check that it uses an even multiple of the
  ! number of PEs in each direction.
  io_layout(:) = (/ 1, 1 /)
  call get_param(param_file, mdl, trim(io_layout_nm), io_layout, &
                 "The processor layout to be used, or 0,0 to automatically \n"//&
                 "set the io_layout to be the same as the layout.", default=1, &
                 layoutparam=.true.)

  if (io_layout(1) < 0) then
    write(mesg,'("MOM_domains_init: IO_LAYOUT(1) = ",i4,".  Negative values "//&
         &"are not allowed in ")') io_layout(1)
    call mom_error(fatal, mesg//trim(io_layout_nm))
  elseif (io_layout(1) > 0) then ; if (modulo(layout(1), io_layout(1)) /= 0) then
    write(mesg,'("MOM_domains_init: The x-direction I/O-layout, IO_LAYOUT(1)=",i4, &
         &", does not evenly divide the x-direction layout, NIPROC=,",i4,".")') &
          io_layout(1),layout(1)
    call mom_error(fatal, mesg)
  endif ; endif

  if (io_layout(2) < 0) then
    write(mesg,'("MOM_domains_init: IO_LAYOUT(2) = ",i4,".  Negative values "//&
         &"are not allowed in ")') io_layout(2)
    call mom_error(fatal, mesg//trim(io_layout_nm))
  elseif (io_layout(2) /= 0) then ; if (modulo(layout(2), io_layout(2)) /= 0) then
    write(mesg,'("MOM_domains_init: The y-direction I/O-layout, IO_LAYOUT(2)=",i4, &
         &", does not evenly divide the y-direction layout, NJPROC=,",i4,".")') &
          io_layout(2),layout(2)
    call mom_error(fatal, mesg)
  endif ; endif

  if (io_layout(2) == 0) io_layout(2) = layout(2)
  if (io_layout(1) == 0) io_layout(1) = layout(1)

  x_flags = 0 ; y_flags = 0
  if (reentrant_x) x_flags = cyclic_global_domain
  if (reentrant_y) y_flags = cyclic_global_domain
  if (tripolar_n) then
    y_flags = fold_north_edge
    if (reentrant_y) call mom_error(fatal,"MOM_domains: "// &
      "TRIPOLAR_N and REENTRANT_Y may not be defined together.")
  endif

  global_indices(1) = 1 ; global_indices(2) = mom_dom%niglobal
  global_indices(3) = 1 ; global_indices(4) = mom_dom%njglobal

  if (mask_table_exists) then
    call mom_define_domain( global_indices, layout, mom_dom%mpp_domain, &
                xflags=x_flags, yflags=y_flags, &
                xhalo=mom_dom%nihalo, yhalo=mom_dom%njhalo, &
                symmetry = mom_dom%symmetric, name=dom_name, &
                maskmap=mom_dom%maskmap )
  else
    call mom_define_domain( global_indices, layout, mom_dom%mpp_domain, &
                xflags=x_flags, yflags=y_flags, &
                xhalo=mom_dom%nihalo, yhalo=mom_dom%njhalo, &
                symmetry = mom_dom%symmetric, name=dom_name)
  endif

  if ((io_layout(1) > 0) .and. (io_layout(2) > 0) .and. &
      (layout(1)*layout(2) > 1)) then
    call mom_define_io_domain(mom_dom%mpp_domain, io_layout)
  endif

! Save the extra data for creating other domains of different resolution that overlay this domain
  mom_dom%X_FLAGS = x_flags
  mom_dom%Y_FLAGS = y_flags
  mom_dom%layout = layout
  mom_dom%io_layout = io_layout
  mom_dom%use_io_layout = (io_layout(1) + io_layout(2) > 0)

  if (is_static) then
  !   A requirement of equal sized compute domains is necessary when STATIC_MEMORY_
  ! is used.
    call mpp_get_compute_domain(mom_dom%mpp_domain,isc,iec,jsc,jec)
    xsiz = iec - isc + 1
    ysiz = jec - jsc + 1
    if (xsiz*niproc /= mom_dom%niglobal .OR. ysiz*njproc /= mom_dom%njglobal) then
       write( char_xsiz,'(i4)' ) niproc
       write( char_ysiz,'(i4)' ) njproc
       write( char_niglobal,'(i4)' ) mom_dom%niglobal
       write( char_njglobal,'(i4)' ) mom_dom%njglobal
       call mom_error(warning,'MOM_domains: Processor decomposition (NIPROC_,NJPROC_) = (' &
           //trim(char_xsiz)//','//trim(char_ysiz)// &
           ') does not evenly divide size set by preprocessor macro ('&
           //trim(char_niglobal)//','//trim(char_njglobal)// '). ')
       call mom_error(fatal,'MOM_domains:  #undef STATIC_MEMORY_ in "//trim(inc_nm)//" to use &
           &dynamic allocation, or change processor decomposition to evenly divide the domain.')
    endif
  endif

end subroutine mom_domains_init

! #@# This subroutine needs a doxygen description and comments.
subroutine clone_md_to_md(MD_in, MOM_dom, min_halo, halo_size, symmetric, &
                          domain_name)
  type(mom_domain_type),           intent(in)    :: MD_in
  type(mom_domain_type),           pointer       :: MOM_dom
  integer, dimension(2), optional, intent(inout) :: min_halo
  integer,               optional, intent(in)    :: halo_size
  logical,               optional, intent(in)    :: symmetric
  character(len=*),      optional, intent(in)    :: domain_name

  integer :: global_indices(4)
  logical :: mask_table_exists
  character(len=64) :: dom_name

  if (.not.associated(mom_dom)) then
    allocate(mom_dom)
    allocate(mom_dom%mpp_domain)
  endif

! Save the extra data for creating other domains of different resolution that overlay this domain
  mom_dom%niglobal = md_in%niglobal ; mom_dom%njglobal = md_in%njglobal
  mom_dom%nihalo = md_in%nihalo ; mom_dom%njhalo = md_in%njhalo

  mom_dom%symmetric = md_in%symmetric
  mom_dom%nonblocking_updates = md_in%nonblocking_updates

  mom_dom%X_FLAGS = md_in%X_FLAGS ; mom_dom%Y_FLAGS = md_in%Y_FLAGS
  mom_dom%layout(:) = md_in%layout(:) ; mom_dom%io_layout(:) = md_in%io_layout(:)
  mom_dom%use_io_layout = (mom_dom%io_layout(1) + mom_dom%io_layout(2) > 0)

  if (associated(md_in%maskmap)) then
    mask_table_exists = .true.
    allocate(mom_dom%maskmap(mom_dom%layout(1), mom_dom%layout(2)))
    mom_dom%maskmap(:,:) = md_in%maskmap(:,:)
  else
    mask_table_exists = .false.
  endif

  if (present(halo_size) .and. present(min_halo)) call mom_error(fatal, &
      "clone_MOM_domain can not have both halo_size and min_halo present.")

  if (present(min_halo)) then
    mom_dom%nihalo = max(mom_dom%nihalo, min_halo(1))
    min_halo(1) = mom_dom%nihalo
    mom_dom%njhalo = max(mom_dom%njhalo, min_halo(2))
    min_halo(2) = mom_dom%njhalo
  endif

  if (present(halo_size)) then
    mom_dom%nihalo = halo_size ; mom_dom%njhalo = halo_size
  endif

  if (present(symmetric)) then ; mom_dom%symmetric = symmetric ; endif

  dom_name = "MOM"
  if (present(domain_name)) dom_name = trim(domain_name)

  global_indices(1) = 1 ; global_indices(2) = mom_dom%niglobal
  global_indices(3) = 1 ; global_indices(4) = mom_dom%njglobal
  if (mask_table_exists) then
    call mom_define_domain( global_indices, mom_dom%layout, mom_dom%mpp_domain, &
                xflags=mom_dom%X_FLAGS, yflags=mom_dom%Y_FLAGS, &
                xhalo=mom_dom%nihalo, yhalo=mom_dom%njhalo, &
                symmetry = mom_dom%symmetric, name=dom_name, &
                maskmap=mom_dom%maskmap )
  else
    call mom_define_domain( global_indices, mom_dom%layout, mom_dom%mpp_domain, &
                xflags=mom_dom%X_FLAGS, yflags=mom_dom%Y_FLAGS, &
                xhalo=mom_dom%nihalo, yhalo=mom_dom%njhalo, &
                symmetry = mom_dom%symmetric, name=dom_name)
  endif

  if ((mom_dom%io_layout(1) + mom_dom%io_layout(2) > 0) .and. &
      (mom_dom%layout(1)*mom_dom%layout(2) > 1)) then
    call mom_define_io_domain(mom_dom%mpp_domain, mom_dom%io_layout)
  endif

end subroutine clone_md_to_md

! #@# This subroutine needs a doxygen description and comments.
subroutine clone_md_to_d2d(MD_in, mpp_domain, min_halo, halo_size, symmetric, &
                           domain_name)
  type(mom_domain_type),           intent(in)    :: MD_in
  type(domain2d),                  intent(inout) :: mpp_domain
  integer, dimension(2), optional, intent(inout) :: min_halo
  integer,               optional, intent(in)    :: halo_size
  logical,               optional, intent(in)    :: symmetric
  character(len=*),      optional, intent(in)    :: domain_name

  integer :: global_indices(4), layout(2), io_layout(2)
  integer :: X_FLAGS, Y_FLAGS, niglobal, njglobal, nihalo, njhalo
  logical :: symmetric_dom
  character(len=64) :: dom_name

! Save the extra data for creating other domains of different resolution that overlay this domain
  niglobal = md_in%niglobal ; njglobal = md_in%njglobal
  nihalo = md_in%nihalo ; njhalo = md_in%njhalo

  symmetric_dom = md_in%symmetric

  x_flags = md_in%X_FLAGS ; y_flags = md_in%Y_FLAGS
  layout(:) = md_in%layout(:) ; io_layout(:) = md_in%io_layout(:)

  if (present(halo_size) .and. present(min_halo)) call mom_error(fatal, &
      "clone_MOM_domain can not have both halo_size and min_halo present.")

  if (present(min_halo)) then
    nihalo = max(nihalo, min_halo(1))
    njhalo = max(njhalo, min_halo(2))
    min_halo(1) = nihalo ; min_halo(2) = njhalo
  endif

  if (present(halo_size)) then
    nihalo = halo_size ; njhalo = halo_size
  endif

  if (present(symmetric)) then ; symmetric_dom = symmetric ; endif

  dom_name = "MOM"
  if (present(domain_name)) dom_name = trim(domain_name)

  global_indices(1) = 1 ; global_indices(2) = niglobal
  global_indices(3) = 1 ; global_indices(4) = njglobal
  if (associated(md_in%maskmap)) then
    call mom_define_domain( global_indices, layout, mpp_domain, &
                xflags=x_flags, yflags=y_flags, &
                xhalo=nihalo, yhalo=njhalo, &
                symmetry = symmetric, name=dom_name, &
                maskmap=md_in%maskmap )
  else
    call mom_define_domain( global_indices, layout, mpp_domain, &
                xflags=x_flags, yflags=y_flags, &
                xhalo=nihalo, yhalo=njhalo, &
                symmetry = symmetric, name=dom_name)
  endif

  if ((io_layout(1) + io_layout(2) > 0) .and. &
      (layout(1)*layout(2) > 1)) then
    call mom_define_io_domain(mpp_domain, io_layout)
  endif

end subroutine clone_md_to_d2d

! #@# This subroutine needs a doxygen description
subroutine get_domain_extent(Domain, isc, iec, jsc, jec, isd, ied, jsd, jed, &
                             isg, ieg, jsg, jeg, idg_offset, jdg_offset, &
                             symmetric, local_indexing, index_offset)
  type(mom_domain_type), &
           intent(in)  :: Domain
  integer, intent(out) :: isc, iec, jsc, jec     !< The start & end indices of the computational
                                                 !! domain.
  integer, intent(out) :: isd, ied, jsd, jed     !< The start & end indices of the data domain.
  integer, intent(out) :: isg, ieg, jsg, jeg     !< The start & end indices of the global domain.
  integer, intent(out) :: idg_offset, jdg_offset !< The offset between the corresponding global and
                                                 !! data index spaces.
  logical, intent(out) :: symmetric              !< True if symmetric memory is used.
  logical, optional, &
            intent(in) :: local_indexing         !< If true, local tracer array indices start at 1,
                                                 !! as in most MOM6 or GOLD code.
  integer, optional, &
            intent(in) :: index_offset           !< A fixed additional offset to all indices. This
                                                 !! can be useful for some types of debugging with
                                                 !! dynamic memory allocation.

! Arguments: Domain - The MOM_domain_type from which the indices are extracted.
!  (out)     isc, iec, jsc, jec - the start & end indices of the
!                                 computational domain.
!  (out)     isd, ied, jsd, jed - the start & end indices of the data domain.
!  (out)     isg, ieg, jsg, jeg - the start & end indices of the global domain.
!  (out)     idg_offset, jdg_offset - the offset between the corresponding
!                                     global and data index spaces.
!  (out)     symmetric - true if symmetric memory is used.
!  (in,opt)  local_indexing - if true, local tracer array indices start at 1, as
!                             in most MOM6 or GOLD code.
!  (in,opt)  index_offset - A fixed additional offset to all indices.  This can
!                           be useful for some types of debugging with dynamic
!                           memory allocation.

  integer :: ind_off
  logical :: local
  local = .true. ; if (present(local_indexing)) local = local_indexing
  ind_off = 0 ; if (present(index_offset)) ind_off = index_offset

  call mpp_get_compute_domain(domain%mpp_domain, isc, iec, jsc, jec)
  call mpp_get_data_domain(domain%mpp_domain, isd, ied, jsd, jed)
  call mpp_get_global_domain(domain%mpp_domain, isg, ieg, jsg, jeg)

  ! This code institutes the MOM convention that local array indices start at 1.
  if (local) then
    idg_offset = isd-1 ; jdg_offset = jsd-1
    isc = isc-isd+1 ; iec = iec-isd+1 ; jsc = jsc-jsd+1 ; jec = jec-jsd+1
    ied = ied-isd+1 ; jed = jed-jsd+1
    isd = 1 ; jsd = 1
  else
    idg_offset = 0 ; jdg_offset = 0
  endif
  if (ind_off /= 0) then
    idg_offset = idg_offset + ind_off ; jdg_offset = jdg_offset + ind_off
    isc = isc + ind_off ; iec = iec + ind_off
    jsc = jsc + ind_off ; jec = jec + ind_off
    isd = isd + ind_off ; ied = ied + ind_off
    jsd = jsd + ind_off ; jed = jed + ind_off
  endif
  symmetric = domain%symmetric

end subroutine get_domain_extent

end module mom_domains