!===============================================================================
!
! place_process_linux.F90
!
!-------------------------------------------------------------------------------
!
! Copyright (C) 2008 Hinnerk Stueben
!
! This file is part of BQCD -- Berlin Quantum ChromoDynamics program
!
! BQCD is free software: you can redistribute it and/or modify
! it under the terms of the GNU General Public License as published by
! the Free Software Foundation, either version 3 of the License, or
! (at your option) any later version.
!
! BQCD is distributed in the hope that it will be useful,
! but WITHOUT ANY WARRANTY; without even the implied warranty of
! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! GNU General Public License for more details.
!
! You should have received a copy of the GNU General Public License
! along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
!
!-------------------------------------------------------------------------------
# include "defs.h"

!-------------------------------------------------------------------------------
subroutine place_process()

  use module_function_decl
  implicit none

  character(16)   :: bqcd_ppn        ! processes per node
  character(1024) :: bqcd_cpu_mapping
  character(32)   :: keyword

  integer, dimension(:), allocatable :: cpu_mapping
  integer        :: ppn
  integer        :: cpus_per_node
  integer        :: iostat
  integer        :: process
  integer        :: cpu
  integer        :: ncpus
  integer        :: i


  call getenv("BQCD_PPN", bqcd_ppn)
  call getenv("BQCD_CPU_MAPPING", bqcd_cpu_mapping)  

  if (bqcd_ppn == " ") return


  ! get processes per node:

  read(bqcd_ppn, *) ppn


  ! get number of CPUs (cores) per node:

  cpus_per_node = 0

  open(UCPUINFO, file = "/proc/cpuinfo", action = "read")
  do
      read(UCPUINFO, *, iostat = iostat) keyword
      if (iostat /= 0) exit
      if (keyword == "processor") cpus_per_node = cpus_per_node + 1
  enddo
  close(UCPUINFO)  

  CHECK(ppn <= cpus_per_node)
  CHECK(ppn * num_threads() <= cpus_per_node)


  ! get CPU ranks:

  ncpus = 1
  do i = 1, len(bqcd_cpu_mapping)
     if (bqcd_cpu_mapping(i:i) == ',') then
        bqcd_cpu_mapping(i:i) = ' '
        ncpus = ncpus + 1
     endif
  enddo
  
  CHECK(ncpus == ppn * num_threads())

  allocate(cpu_mapping(0:ncpus - 1))
  
  read(bqcd_cpu_mapping, *) cpu_mapping

  do i = 0, ncpus - 1
     CHECK(cpu_mapping(i) < cpus_per_node .and. cpu_mapping(i) >=0)
  enddo

  if (my_pe() == 0) then
     write(STDERR,"(x,a,i8)") "place_process(): ppn          ", ppn
     write(STDERR,"(x,a,i8)") "place_process(): cpus_per_node", cpus_per_node
     write(STDERR,"(x,a,i8)") "place_process(): num_threads  ", num_threads()
     write(STDERR,*)          "place_process(): cpu_mapping  ", cpu_mapping
  endif


  ! place this process (or thread respectively)

  !$omp parallel default(none) &
  !$omp          shared(ppn, cpu_mapping, cpus_per_node) &
  !$omp          private(i, process, cpu)

  !$omp critical

  process = mod(my_pe(), ppn)
  cpu = cpu_mapping(process * num_threads() + my_thread())

  write(STDERR,"(x,a,4i8)") "place_process(): rank/process/thread/cpu", &
                             my_pe(), process, my_thread(), cpu

  CHECK(cpu < cpus_per_node .and. cpu >= 0)

  call place_proc(cpu)

  !$omp end critical
  !$omp end parallel

  deallocate(cpu_mapping)

end

!===============================================================================
