!===============================================================================
!
! ildg_io_mpi_r8.F90
!
!-------------------------------------------------------------------------------
!
! Copyright (C) 2008-2014 Hinnerk Stueben
!
! This file is part of BQCD -- Berlin Quantum ChromoDynamics program
!
! BQCD is free software: you can redistribute it and/or modify
! it under the terms of the GNU General Public License as published by
! the Free Software Foundation, either version 3 of the License, or
! (at your option) any later version.
!
! BQCD is distributed in the hope that it will be useful,
! but WITHOUT ANY WARRANTY; without even the implied warranty of
! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! GNU General Public License for more details.
!
! You should have received a copy of the GNU General Public License
! along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
!
!-------------------------------------------------------------------------------
# include "defs.h"

!-------------------------------------------------------------------------------
subroutine ildg_io_r8(action, u, words_per_site, cksum)

  use module_function_decl
  use module_lattice_io
  use module_vol
  implicit none
  include 'mpif.h'
  character(*), intent(in)       :: action
  integer, intent(in)            :: words_per_site
  real(ILDG_KIND), intent(inout) :: u(words_per_site, 0:NX-1, 0:NY-1, 0:NZ-1, 0:NT-1)
  INT8, intent(out)              :: cksum

  real(ILDG_KIND)    :: buffer(words_per_site, 0:LX - 1, 0:LY - 1)

  integer, parameter :: bytes_per_word = ILDG_KIND

  integer :: x, y, z, t, dx
  integer :: global(DIM), local(DIM), home
  integer :: status(MPI_STATUS_SIZE), ierror
  integer :: words_per_copy
  INT8    :: words_in_u, bytes_in_u
  INT8    :: words_in_buffer, bytes_in_buffer
  INT8    :: bytes_total

  words_in_u = words_per_site
  words_in_u = words_in_u * vol
  bytes_in_u = words_in_u * bytes_per_word

  words_in_buffer = words_per_site
  words_in_buffer = words_in_buffer * LX * LY
  bytes_in_buffer = words_in_buffer * bytes_per_word

  words_per_copy  = words_per_site * NX

  cksum = 0
  bytes_total = 0

  if (action == "write") call swap_endian8(words_in_u, u)

  do t = 0, LT - 1
  do z = 0, LZ - 1

     if (action == "read" .and. my_pe() == 0) then
        call ildg_read(buffer, bytes_in_buffer)
        call cksum2(buffer, bytes_in_buffer, cksum, bytes_total)
     endif

     do y = 0, LY - 1
     do x = 0, LX - 1, NX

        global = (/x, y, z, t/)
        call global2local(global, home, local)

        if (my_pe() == 0) then

           if (home == 0) then

              if (action == "read") then
                 do dx = 0, NX - 1
                    u(:, local(1) + dx, local(2), local(3), local(4)) &
                       = buffer(:, x + dx, y)
                 enddo
              endif

              if (action == "write") then
                 do dx = 0, NX - 1
                    buffer(:, x + dx, y) &
                       = u(:, local(1) + dx, local(2), local(3), local(4))
                 enddo
              endif
           else

              if (action == "read") &
                 call mpi_ssend(buffer(1, x, y), &
                               words_per_copy, ILDG_REAL, home, 0, MPI_COMM_WORLD, ierror)

              if (action == "write") &
                 call mpi_recv(buffer(1, x, y), &
                               words_per_copy, ILDG_REAL, home, 0, MPI_COMM_WORLD, status, ierror)

           endif
        else

           if (home == my_pe()) then

              if (action == "read") &
                 call mpi_recv(u(1, local(1), local(2), local(3), local(4)), & 
                               words_per_copy, ILDG_REAL, 0, 0, MPI_COMM_WORLD, status, ierror)

              if (action == "write") &
                 call mpi_ssend(u(1, local(1), local(2), local(3), local(4)), & 
                               words_per_copy, ILDG_REAL, 0, 0, MPI_COMM_WORLD, ierror)
           endif
        endif

     enddo
     enddo

     if (action == "write" .and. my_pe() == 0) then
        call cksum2(buffer, bytes_in_buffer, cksum, bytes_total)
        call ildg_write(buffer, bytes_in_buffer)
     endif

  enddo
  enddo

  call swap_endian8(words_in_u, u)
  if (my_pe() == 0) call cksum2_end(cksum, bytes_total)
  call mpi_barrier(MPI_COMM_WORLD, ierror)

end

!===============================================================================
