/*
*===============================================================================
*
* d/wilson_d520/neighb.c
*
*-------------------------------------------------------------------------------
*
* Copyright (C) 2010 Thomas Streuer 
*
* This file is part of BQCD -- Berlin Quantum ChromoDynamics program
*
* BQCD is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* BQCD is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with BQCD.  If not, see <http:*www.gnu.org/licenses/>.
*
*-------------------------------------------------------------------------------
*/
#include "common.h"
#include "globals.h"
#include "lattice.h"
#include "memory.h"
#include "neighb.h"

#include <assert.h>
#include <mpi.h>
#include <stdlib.h>

#include <spi/kernel_interface.h>
#include <spi/DMA_Counter.h>
#include <spi/DMA_InjFifo.h>

#include <common/bgp_personality.h>
#include <common/bgp_personality_inlines.h>


static void setup_fifos();

void logical_to_phys(int* l, int* p)
{
  if (t_local)
    for (int mu=0; mu<4; ++mu)
      p[torus_dir[mu]] = l[mu];
  else
  {
    p[0] = l[torus_dir[0]] / 2;
    p[1] = l[torus_dir[1]] / 2;
    p[2] = l[torus_dir[2]];
    p[3] = (l[torus_dir[0]] % 2) + 2 * (l[torus_dir[1]] % 2);
  }
}

void phys_to_logical_to_phys(int* p, int* l)
{
  if (t_local)
    for (int mu=0; mu<4; ++mu)
      l[mu] = p[torus_dir[mu]];
  else
  {
    l[0] = 2 * p[0] + (p[3] % 2);
    l[1] = 2 * p[1] + (p[3] / 2);
    l[2] = p[2];
    l[3] = 0;
  }
}

uint16_t* loc_neighb[2];

void init_lat(int* l, int* g)
{
  MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
  MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
  for (unsigned i=0; i<4; ++i)
    lattice[i] = l[i];
  for (unsigned i=0; i<4; ++i)
    grid[i] = g[i];

  _BGP_Personality_t personality;
  unsigned p_position[4], p_grid[4];
  Kernel_GetPersonality(&personality, sizeof(personality));
  p_position[0] = personality.Network_Config.Xcoord;
  p_position[1] = personality.Network_Config.Ycoord;
  p_position[2] = personality.Network_Config.Zcoord;
  p_position[3] = Kernel_PhysicalProcessorID();

  which_cpu = p_position[3];

  switch(personality.Kernel_Config.ProcessConfig)
  {
  case _BGP_PERS_PROCESSCONFIG_SMP:
    p_grid[3] = 1;
    break;
  case _BGP_PERS_PROCESSCONFIG_2x2:
    p_grid[3] = 2;
    break;
  case _BGP_PERS_PROCESSCONFIG_VNM:
    p_grid[3] = 4;
    break;
  }
  p_grid[0] = personality.Network_Config.Xnodes;
  p_grid[1] = personality.Network_Config.Ynodes;
  p_grid[2] = personality.Network_Config.Znodes;

  mesh_topology = !(BGP_Personality_isTorusX(&personality) && BGP_Personality_isTorusY(&personality) &&
		    BGP_Personality_isTorusZ(&personality));
  if (mesh_topology)
    if (my_rank == 0)
      printf("WARNING: mesh topology\n");


// setup mapping logical <-> physical directions

  unsigned dir;
  unsigned d_rank[4];
  d_rank[0] = 1;
  d_rank[1] = grid[0];
  d_rank[2] = grid[0] * grid[1];
  d_rank[3] = grid[0] * grid[1] * grid[2];
  int dir_used[4] = {0, 0, 0, 0};

  for (unsigned mu=0; mu<4; ++mu)
    torus_dir[mu] = -1;
  /* if (grid[3] == p_grid[3]) */
  /* { */
  /*   torus_dir[3] = 3; */
  /*   dir_used[3] = 1; */
  /* } */
  for (unsigned mu=0; mu<4; ++mu)
    if (torus_dir[mu] == -1 && grid[mu] > 1)
    {
      if (my_rank == d_rank[mu])
      {
	assert(p_position[0] + p_position[1] + p_position[2] + p_position[3] == 1);
	for (unsigned i=0; i<4; ++i)
	  if (p_position[i] == 1)
	    torus_dir[mu] = i;
      }    
      MPI_Bcast(torus_dir + mu, sizeof(torus_dir[0]), MPI_BYTE, d_rank[mu], MPI_COMM_WORLD);
      dir_used[torus_dir[mu]] = 1;
    }
  unsigned n1 = 0;
  for (unsigned mu=0; mu<4; ++mu)
    if (torus_dir[mu] == -1)
    {
      while (dir_used[n1] && n1<3)
	++n1;
      assert(n1 < 4);
      torus_dir[mu] = n1;
      dir_used[n1] = 1;
    }

  if (my_rank == 0)
    printf("PHYS_DIR: %i %i %i %i\n", 
	   torus_dir[0], torus_dir[1], torus_dir[2], torus_dir[3]);
  assert(torus_dir[3] == 3);
  if (grid[torus_dir[3]] == 1)
    t_local = 0;
  else if (grid[torus_dir[3]] == 4)
    t_local = 1;
  else
    die("Invalid value for grid[3]: must be 1 or 4");

  for (unsigned i=0; i<4; ++i)
    logical_dir[torus_dir[i]] = i;
  int global_lattice[4];

  for (unsigned i=0; i<4; ++i)
    global_lattice[i] = lattice[i]*grid[i];

  for (unsigned i=0; i<4; ++i)
    boundary[i] = (grid[i] != 1);

  position[0]  = my_rank % grid[0];
  position[1]  = (my_rank / grid[0]) % grid[1];
  position[2]  = (my_rank / (grid[0] * grid[1] )) % grid[2];
  position[3]  = my_rank / (grid[0] * grid[1] * grid[2]);


  logical_to_phys(position, t_position);

  for (int i=0; i<8; ++i)
    local_nb[0][i] = 0;
  if (t_local)
  {
    local_nb[0][logical_dir[3]] = 1;
    local_nb[1][logical_dir[3]] = 1;
  }
  else
  {
    if (position[logical_dir[0]] % 2 == 0)
      local_nb[0][logical_dir[0]] = 1;
    else
      local_nb[1][logical_dir[0]] = 1;
    if (position[logical_dir[1]] % 2 == 0)
      local_nb[0][logical_dir[1]] = 1;
    else
      local_nb[1][logical_dir[1]] = 1;
  }
//  printf("MPI rank %3i, lpos %2i/%2i/%2i/%2i, tpos %2i/%2i/%2i/%2i, ppos %2i/%2i/%2i/%2i\tlgrid %2i/%2i/%2i/%2i, pgrid %2i/%2i/%2i/%2i\n",
//	 my_rank,
//	 position[0], position[1], position[2], position[3], 
//	 t_position[0], t_position[1], t_position[2], t_position[3], 
//	 p_position[0], p_position[1], p_position[2], p_position[3],
//	 grid[0], grid[1], grid[2], grid[3], 
//	 p_grid[0], p_grid[1], p_grid[2], p_grid[3]);
//

  int ok = 1;
  for (int i=0; i<4; ++i)
    if (t_position[i] != p_position[i])
      ok = 0;
// disabled for testing
  /* for (int i=0; i<4; ++i) */
  /*   if (grid[torus_dir[i]] != p_grid[i]) */
  /*     ok = 0; */
  assert(ok == 1);

  for (unsigned mu=0; mu<4; ++mu)
  {
    int n[4];
    for (unsigned i=0; i<4; ++i)
      n[i] = position[i];

    n[mu] = (position[mu] + 1) % grid[mu];
    grid_nb[0][mu] = n[0] + grid[0] * (n[1] + grid[1] * (n[2] + grid[2] * n[3]));

    n[mu] = (position[mu]+grid[mu]-1) % grid[mu];
    grid_nb[1][mu] = n[0] + grid[0] * (n[1] + grid[1] * (n[2] + grid[2] * n[3]));
  }
  global_oe = (position[0]*lattice[0]+position[1]*lattice[1]+
	       position[2]*lattice[2]+position[3]*lattice[3])%2;
  for (int mu=0; mu<4; ++mu)
    global_oe_nb[mu] = (global_oe+lattice[mu])%2;

  volume = lattice[0] * lattice[1] * lattice[2] * lattice[3];
  half_volume[0] = volume / 2;
  half_volume[1] = volume / 2;
  if (volume % 2)
    ++half_volume[global_oe];
  volume_slice = lattice[0] * lattice[1] * lattice[2];
  half_slice[0] = volume_slice / 2;
  half_slice[1] = volume_slice / 2;
  if (volume % 2)
    ++half_slice[global_oe];

  void* mem;
  if (posix_memalign(&mem, 4096, (sizeof(double) * 18 + sizeof(uint16_t)) * 8 * (half_volume[0] + 2)))
    die("Cannot allocate memory for U[0]");
  wilson_data.w_data[0][0][0].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(double) + sizeof(uint16_t)));
  wilson_data.w_data[0][0][1].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(double) + sizeof(uint16_t)));
  wilson_data.w_data[0][0][0].neighb = mem;
  wilson_data.w_data[0][0][1].neighb = mem;

  if (posix_memalign(&mem, 4096, (sizeof(float) * 18 + sizeof(uint16_t)) * 8 * (half_volume[0] + 2)))
    die("Cannot allocate memory for U[0]");
  wilson_data.w_data[1][0][0].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(float) + sizeof(uint16_t)));
  wilson_data.w_data[1][0][1].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(float) + sizeof(uint16_t)));
  wilson_data.w_data[1][0][0].neighb = mem;
  wilson_data.w_data[1][0][1].neighb = mem;

  if (posix_memalign(&mem, 4096, (sizeof(double) * 18 + sizeof(uint16_t)) * 8 * (half_volume[1] + 2)))
    die("Cannot allocate memory for U[1]");
  wilson_data.w_data[0][1][0].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(double) + sizeof(uint16_t)));
  wilson_data.w_data[0][1][1].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(double) + sizeof(uint16_t)));
  wilson_data.w_data[0][1][0].neighb = mem;
  wilson_data.w_data[0][1][1].neighb = mem;

  if (posix_memalign(&mem, 4096, (sizeof(float) * 18 + sizeof(uint16_t)) * 8 * (half_volume[1] + 2)))
    die("Cannot allocate memory for U[1]");
  wilson_data.w_data[1][1][0].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(float) + sizeof(uint16_t)));
  wilson_data.w_data[1][1][1].U = (double*) ((char*) mem + 2 * 8 * (18 * sizeof(float) + sizeof(uint16_t)));
  wilson_data.w_data[1][1][0].neighb = mem;
  wilson_data.w_data[1][1][1].neighb = mem;

// init neighbor array  (2 extra sites because adirac loads ahead)
  unsigned neighb_offs[2];
  neighb_offs[0] = 8 * 18 * sizeof(double) / sizeof(uint16_t) + 8;
  neighb_offs[1] = 8 * 18 * sizeof(float) / sizeof(uint16_t) + 8;

  for (int a=0; a<2; ++a)
  {
    loc_neighb[a] = malloc((8 * half_volume[a] + 16) * sizeof (*loc_neighb[a]));
    assert(loc_neighb[a]);
    init_nb_list(loc_neighb[a], a);
    if (T0_LAST)
    {
      uint16_t* tmp_nb = malloc(sizeof(*tmp_nb) * half_slice[a] * 8);
      assert(tmp_nb);
      for (unsigned x=0; x<8*half_slice[a]; ++x)
	tmp_nb[x] = loc_neighb[a][x];
      for (unsigned x=0; x<8*(half_volume[a]-half_slice[a]); ++x)
	loc_neighb[a][x] = loc_neighb[a][x + 8 * half_slice[a]];
      for (unsigned x=0; x<8*half_slice[a]; ++x)
	loc_neighb[a][x + 8 * (half_volume[a] - half_slice[a])] = tmp_nb[x];
      free(tmp_nb);
      unsigned b_ctr = 0;
      uint16_t* nb_ptr = loc_neighb[a];
      for (int x=0; x<8*half_volume[a]; ++x, ++nb_ptr)
	if (*nb_ptr & (1 <<  NB_ARRAY_RECV_BIT))
	  *nb_ptr = (3 * b_ctr++ ) | (1 <<  NB_ARRAY_RECV_BIT);
      assert(3*b_ctr < (1 << 16));
    }
// make sure that adirac always loads valid data (but no remote data (counters)
    for (unsigned i=0; i<2*8; ++i)
      loc_neighb[a][8 * half_volume[a] + i] = 0;
  }

// initialize shadow counters
  for (unsigned a=0; a<8; ++a)
  {
    unsigned ictr[8];
    for (unsigned i=0; i<8; ++i)
    {
      ictr[i] = 0;
      if (loc_neighb[(a / 2) % 2][i    ] &  1 << NB_ARRAY_RECV_BIT) // remote load in it. 0 ?
	++ictr[i];
      if (loc_neighb[(a / 2) % 2][i + 8] &  1 << NB_ARRAY_RECV_BIT) // remote load in it. 1 ?
	++ictr[i];
    }
    for (unsigned i=0; i<4; ++i)
      wilson_data.w_data[0][0][a].ctr_init[i] = 3 * (ictr[2 * i] << 16 | ictr[2 * i + 1]);
  }

  for (unsigned a=0; a<8; ++a)
  {
    int oe = (a / 2) % 2;
    wilson_data.w_data[0][0][a].volh   = half_volume[oe];
    wilson_data.w_data[0][0][a].volh3  = half_slice[oe];
  }
  for (unsigned a=0; a<4; ++a)
  {
    wilson_data.w_data[0][0][a].pref_offset   = (volume_slice + 8) * 12 * sizeof(double);
    wilson_data.w_data[1][0][a].pref_offset   = (volume_slice + 8) * 12 * sizeof(float);
  }

  setup_fifos();

  if (mesh_topology && !mesh_correct)
  {
    for (int oe=0; oe<2; ++oe)
    {
      uint16_t* nb = loc_neighb[oe];
      for (int x=0; x<half_volume[oe]; ++x, nb+=8)
	for (int mu=0; mu<4; ++mu)
	  if (torus_dir[mu] != 3 || use_shmem)
	  {
	    if (position[mu] == 0)
	      nb[4 + mu] &=  ~(1 << NB_ARRAY_RECV_BIT);
	    if (position[mu] == grid[mu] - 1)
	      nb[    mu] &=  ~(1 << NB_ARRAY_RECV_BIT);
	  }
    }

    for (int a=0; a<8; ++a)
      for (int mu=0; mu<4; ++mu)
	if (torus_dir[mu] != 3)
	{
	  if (position[mu] == 0)
	  {
	    wilson_data.w_data[0][0][a].fifo_size[4 + mu] = 0;
	    wilson_data.w_data[0][0][a].ctr_init[2 + mu / 2] &= 0xffff << (16 * (mu % 2));
	  }
	  if (position[mu] == grid[mu] - 1)
	  {
	    wilson_data.w_data[0][0][a].fifo_size[mu] = 0;
	    wilson_data.w_data[0][0][a].ctr_init[mu / 2] &= 0xffff << (16 * (mu % 2));
	  }
	}
  }

  for (int oe=0; oe<2; ++oe)
    for (unsigned x=0; x<half_volume[oe]+2; ++x)
      for (unsigned i=0; i<8; ++i)
	for (int sd=0; sd<2; ++sd)
	  wilson_data.w_data[sd][oe][0].neighb[neighb_offs[sd] * x + i] = loc_neighb[oe][8 * x + i];

  wilson_data.mod2_ctr = 1;
}

static void setup_fifos()
{
// allocate inj counters
  int n_sg;
  int sg[DMA_NUM_COUNTER_SUBGROUPS_PER_GROUP];
  int r = DMA_CounterGroupQueryFree(
    DMA_Type_Injection, which_cpu, &n_sg, sg);
  if (r)
    die("Error DMA_CounterGroupQueryFree (inj)");
  if (n_sg < 1)
    die("No injection counters available");
  int inj_cg_id = *sg;
  DMA_CounterGroup_t injection_cg;
  r = DMA_CounterGroupAllocate(
    DMA_Type_Injection, which_cpu, 1, &inj_cg_id, 0, 0, 0, 0, &injection_cg);
  if (r)
    die("Error DMA_CounterGroupAllocate (inj)");
  for (unsigned i=1; i<8; ++i)
    assert(injection_cg.counter[8*inj_cg_id  ].counter_hw_ptr + i ==
	   injection_cg.counter[8*inj_cg_id+i].counter_hw_ptr);
  for (unsigned a=0; a<8; ++a)
    wilson_data.w_data[0][0][a].inj_counter_hw = injection_cg.counter[8*inj_cg_id].counter_hw_ptr;

// allocate rec counters

  r = DMA_CounterGroupQueryFree(
    DMA_Type_Reception, which_cpu, &n_sg, sg);
  if (r)
    die("Error DMA_CounterGroupQueryFree (rec)");
  if (n_sg < 1)
    die("No reception counters available");
  int rec_cg_id = *sg;
  DMA_CounterGroup_t reception_cg;
  r = DMA_CounterGroupAllocate(
    DMA_Type_Reception, which_cpu, 1, &rec_cg_id, 0, 0, 0, 0, &reception_cg);
  if (r)
    die("Error DMA_CounterGroupAllocate (rec)");
  for (unsigned i=1; i<8; ++i)
    assert(reception_cg.counter[8*rec_cg_id  ].counter_hw_ptr + i ==
	   reception_cg.counter[8*rec_cg_id+i].counter_hw_ptr);
  for (unsigned a=0; a<8; ++a)
    wilson_data.w_data[0][0][a].rec_counter_hw = reception_cg.counter[8*rec_cg_id].counter_hw_ptr;

  int rec_cg_id_nb[2][4];
  for (unsigned fw=0; fw<2; ++fw)
    for (unsigned mu=0; mu<4; ++mu)
      if (boundary[mu])
      {
	MPI_Status stat;
	MPI_Sendrecv(
	  &rec_cg_id, sizeof(rec_cg_id), MPI_BYTE, grid_nb[1-fw][mu], 0,
	  &rec_cg_id_nb[fw][mu], sizeof(rec_cg_id), MPI_BYTE, grid_nb[fw][mu], 0,
	  MPI_COMM_WORLD, &stat);
      }

// allocate fifos
// 16 fifos: 0..7 "even", 8..15 "odd"
  int n_fifo;
  int free_fifo_id[DMA_NUM_INJ_FIFOS_PER_GROUP];
  r = DMA_InjFifoGroupQueryFree(which_cpu, &n_fifo, free_fifo_id);
  if (r)
    die("Error DMA_InjFifoGroupQueryFree");
  if (n_fifo < 8)
    die("Not enough fifos available");
 
  unsigned short prio[8];
  unsigned short local[8];
  unsigned char map[8];
  int fifo_id[8];
  for (unsigned i=0; i<8; ++i)
  {
    fifo_id[i] = free_fifo_id[i];
    prio[i]    = 0;
    local[i]   = local_nb[0][i];
    if (local[i])
      map[i] = 0;
    else
      map[i] = 1 << (2 * which_cpu + (i/4));
  }
  DMA_InjFifoGroup_t injection_fg;
  r = DMA_InjFifoGroupAllocate(which_cpu, 8, fifo_id, prio, local, map,
			       0, 0, 0, 0, 0, &injection_fg);
  if (r)
    die("Error DMA_InjFifoGroupAllocate");

  unsigned total_bnd_size[2];
  
  for (unsigned oe=0; oe<2; ++oe)
  {
    total_bnd_size[oe] - 0;
    for (unsigned i=0; i<8*half_volume[oe]; ++i)
      if (loc_neighb[oe][i] & (1 << NB_ARRAY_RECV_BIT))
	++total_bnd_size[oe];
  }
  unsigned max_total_bnd_size = total_bnd_size[0] > total_bnd_size[1] ? 
    total_bnd_size[0] : total_bnd_size[1];
  double* comm_buf;
  if (posix_memalign((void**) &comm_buf, 128, 2 * max_total_bnd_size * 24 * sizeof (*comm_buf)))
    die("Cannot allocate memory (comm_buf)");
  
  for (int a=0; a<4; ++a)
  {
    wilson_data.w_data[0][a][0].comm_buffer = comm_buf;
    wilson_data.w_data[0][a][1].comm_buffer = comm_buf + 24 * max_total_bnd_size;
  }

  for (unsigned i=0; i<8; ++i)
  {
    DMA_CounterSetValue(&(injection_cg.counter[8*inj_cg_id+i]), 0);
    DMA_CounterSetEnableById(&injection_cg, 8*inj_cg_id+i);
  }
  for (unsigned i=0; i<8; ++i)
  {
    r = DMA_CounterSetValueBaseMax(&(reception_cg.counter[8*rec_cg_id+i]), -1, comm_buf,
    				   comm_buf + 2 * max_total_bnd_size * 24 * sizeof (*comm_buf));
    if (r)
      die("Error DMA_CounterSetValueBaseMax (rec, i=%i)", i);
    DMA_CounterSetEnableById(&reception_cg, 8*rec_cg_id+i);
  }

  uint16_t* nb_neighb[2][4];
  for (unsigned i=0; i<8; ++i)
  {
    nb_neighb[0][i] = malloc(4 * (volume + 1) * sizeof (*(nb_neighb[0][i])));
    if (!nb_neighb[0][i])
      die("Cannot allocate memory (nb_neighb)");
  }

  unsigned desc_count[2][8];
  for (unsigned oe=0; oe<2; ++oe)
  {
    uint16_t* nb = loc_neighb[1-oe];
    for (unsigned i=0; i<8; ++i)
    {
      desc_count[oe][i] = 0;
      for (unsigned x=0; x<half_volume[1-oe]; ++x)
	if (nb[8 * x + i] & (1 << NB_ARRAY_RECV_BIT))
	  ++desc_count[oe][i];
      wilson_data.w_data[0][oe][0].fifo_size[i] = desc_count[oe][i] * 24 * sizeof(double);
      wilson_data.w_data[0][oe][1].fifo_size[i] = desc_count[oe][i] * 24 * sizeof(double);
      wilson_data.w_data[1][oe][0].fifo_size[i] = desc_count[oe][i] * 24 * sizeof(float);
      wilson_data.w_data[1][oe][1].fifo_size[i] = desc_count[oe][i] * 24 * sizeof(float);
    }
  }


// allocate memory for fifos

  DMA_InjDescriptor_t* fifo[2][2][2][8];
  for (int sd=0; sd<2; ++sd)
    for (unsigned i=0; i<8; ++i)
    {
      unsigned n = desc_count[0][i] + desc_count[1][i] + 2;
      if (n > 2 && (desc_count[0][i] == 0 || desc_count[1][i] == 0))
	die("Invalid setup (%i, %i, %i, %i)",
	    i, n, desc_count[0][i], desc_count[1][i]);
      if (posix_memalign((void**) &(fifo[sd][0][0][i]), 128, 2 * n * sizeof (DMA_InjDescriptor_t)))
	die("Cannot allocate memory for fifo");
      for (unsigned a=0; a<2; ++a)
      {
	fifo[sd][0][1][i] = fifo[sd][0][0][i] + 1 + desc_count[0][i];
	fifo[sd][1][0][i] = fifo[sd][0][0][i] + 2 + 2 * desc_count[0][i];
	fifo[sd][1][1][i] = fifo[sd][0][0][i] + 1 + n + desc_count[0][i];
      }
    }
  
  for (unsigned oe=0; oe<2; ++oe)
  {
    for (unsigned fw=0; fw<2; ++fw)
      for (unsigned mu=0; mu<4; ++mu)
	if (boundary[mu])
	{
	  MPI_Status stat;
	  MPI_Sendrecv(
	    loc_neighb[oe], 8 * sizeof(*(nb_neighb[0][0])) * half_volume[oe], MPI_BYTE,
	    grid_nb[1-fw][mu], 0,
	    nb_neighb[fw][mu], 8 * sizeof(*(nb_neighb[0][0])) * half_volume[1-oe], MPI_BYTE,
	    grid_nb[fw][mu], 0,
	    MPI_COMM_WORLD, &stat);
	}

    for (unsigned fw=0; fw<2; ++fw)
      for (unsigned mu=0; mu<4; ++mu)
      {
	if (torus_dir[mu] != 3 || !use_shmem)
	{
	  unsigned hint =
	    fw == 0 && torus_dir[mu] == 0 ? DMA_PACKET_HINT_XP :
	    fw == 0 && torus_dir[mu] == 1 ? DMA_PACKET_HINT_YP :
	    fw == 0 && torus_dir[mu] == 2 ? DMA_PACKET_HINT_ZP :
	    fw == 1 && torus_dir[mu] == 0 ? DMA_PACKET_HINT_XM :
	    fw == 1 && torus_dir[mu] == 1 ? DMA_PACKET_HINT_YM :
	    fw == 1 && torus_dir[mu] == 2 ? DMA_PACKET_HINT_ZM : 0;
	  if (mesh_topology && mesh_correct)
	    hint = 0;
	  int l_dest[4];
	  for (unsigned i=0; i<4; ++i)
	    l_dest[i] = position[i];
	  if (fw == 0)
	    l_dest[mu] = (l_dest[mu] + 1) % grid[mu];
	  else
	    l_dest[mu] = (l_dest[mu] + grid[mu] - 1) % grid[mu];
	  int dest[4]; // hw coordinates of destination
	  logical_to_phys(l_dest, dest);
	  unsigned fwmu = 4 * fw + mu;
	  uint16_t* nb = loc_neighb[1-oe];
	  unsigned* dest_pos = malloc(desc_count[oe][fwmu] * sizeof(*dest_pos));
	  if (!dest_pos)
	    die("Cannot allocate memory (dest_pos)");
	  unsigned* dp_ptr = dest_pos;
	  unsigned p = 0;
	  for (unsigned x=0; x<half_volume[1-oe]; ++x)
	    for (unsigned fw1=0; fw1<2; ++fw1)
	      for (unsigned mu1=0; mu1<4; ++mu1)
		if (nb_neighb[fw][mu][8 * x + 4*fw1 + mu1] & (1 << NB_ARRAY_RECV_BIT))
		{
		  if (fw1 == 1-fw && mu1 == mu)
		    *dp_ptr++ = p;
		  ++p;
		}
	  assert(dp_ptr - desc_count[oe][fwmu] == dest_pos);

// initialize descriptors
	  for (int sd=0; sd<2; ++sd)
	  {
	    int basic_size = sd==0? sizeof(double) : sizeof(float);
	    for (int a=0; a<2; ++a)
	    {
	      DMA_InjDescriptor_t* fifo_ptr = fifo[sd][oe][a][fwmu];
	      p = 0;
	      for (unsigned x=0; x<half_volume[1-oe]; ++x)
	      {
		unsigned xr;
		if (!T0_LAST)
		  xr = x;
		else
		  xr = (x + half_slice[1-oe]) % half_volume[1-oe];
		if (nb[8 * x + fwmu] & (1 << NB_ARRAY_RECV_BIT))
		{
		  if (!local_nb[fw][mu])
		  { 
		    int r = DMA_TorusDirectPutDescriptor(
		      fifo_ptr,
		      dest[0], dest[1], dest[2],               // destination
		      hint,                                    // hint bits
		      DMA_PACKET_VC_BN,                        // virtual channel
		      which_cpu,                               // injection counter group
		      8 * inj_cg_id + fwmu,                    // injection counter id
		      xr * 24 * basic_size,                    // send offs
		      dest[3],                                 // reception counter group
		      8 * rec_cg_id_nb[0][fwmu] + (fwmu ^ 4),  // reception counter id
		      a * max_total_bnd_size * 24 * basic_size +
		      dest_pos[p++] * 24 * basic_size,         // recv offs
		      24 * basic_size);                        // msg len
		    if (r)
		      die("Error DMA_TorusDirectPutDescriptor");
		  }
		  else
		  { 
		    int r = DMA_LocalDirectPutDescriptor(
		      fifo_ptr,
		      which_cpu,                               // injection counter group
		      8 * inj_cg_id + fwmu,                    // injection counter id
		      xr * 24 * basic_size,                    // send offs
		      dest[3],                                 // reception counter group
		      8 * rec_cg_id_nb[0][fwmu] + (fwmu ^ 4),  // reception counter id
		      a * max_total_bnd_size * 24 * basic_size +
		      dest_pos[p++] * 24 * basic_size,         // recv offs
		      24 * basic_size);                        // msg len
		    if (r)
		      die("Error DMA_LocalDirectPutDescriptor");
		  }
		  ++fifo_ptr;
		}
	      }
// add "guard" descriptor
	      int r = DMA_LocalDirectPutDescriptor(
		fifo_ptr,
		which_cpu,                               // injection counter group
		8 * inj_cg_id + fwmu,                    // injection counter id
		0,                                       // send offs
		dest[3],                                 // reception counter group
		8 * rec_cg_id_nb[0][fwmu] + (fwmu ^ 4),  // reception counter id
		0,                                       // recv offs
		192);                                    // msg len
	      if (r)
		die("Error DMA_LocalDirectPutDescriptor");
	      fifo_ptr->msg_length = 0;                  // msg len=0 --> will abort
	    }
	  }
	  free(dest_pos);
	  }
      }
  }
  for (unsigned i=0; i<8; ++i)
    free(nb_neighb[0][i]);

  MPI_Barrier(MPI_COMM_WORLD);

  for (unsigned i=0; i<8; ++i)
    if (i%4 != 3 || !use_shmem)
    {
      unsigned sz = 4 * (desc_count[0][i] + desc_count[1][i] + 2);
      if (sz > 8)
      {
	const unsigned min_sz = 32;
	if (sz < min_sz)
	  sz = min_sz;
	int r = DMA_InjFifoInitById(&injection_fg, fifo_id[i],
				    fifo[0][0][0][i],
				    fifo[0][0][0][i],
				    fifo[0][0][0][i] + sz);
	if (r)
	  die("Cannot initbyid: %i, %i", i, r);
      }
    }
  DMA_FifoHW_t* dummy_fifo_hw = malloc(sizeof(*dummy_fifo_hw));
  assert(dummy_fifo_hw != 0);
  for (unsigned i=0; i<8; ++i)
  {
    int bnd = boundary[i % 4];
    if (mesh_topology && !mesh_correct && torus_dir[i % 4] != 3)
    {
      if (i < 4 && position[i % 4] == grid[i % 4] - 1)
	bnd = 0;
      if (i > 3 && position[i % 4] == 0)
	bnd = 0;
    }
    if (torus_dir[i % 4] == 3 && use_shmem)
      bnd = 0;
    if (!bnd)
    {
      for (unsigned a=0; a<8; ++a)
      {
	wilson_data.w_data[0][0][a].fifo_hw[i]    = dummy_fifo_hw;
	wilson_data.w_data[0][0][a].pa_head[i]    = 0;
	wilson_data.w_data[0][0][a].pa_tail[i]    = 0;
	wilson_data.w_data[0][0][a].fifo_size[i]  = 0;
      }
    }
    else
    {
      for (unsigned a=0; a<8; ++a)
	wilson_data.w_data[0][0][a].fifo_hw[i]  = injection_fg.fifos[fifo_id[i]].dma_fifo.fifo_hw_ptr;
      wilson_data.w_data[0][0][0].pa_head[i]  = injection_fg.fifos[fifo_id[i]].dma_fifo.pa_start;
      wilson_data.w_data[0][0][0].pa_tail[i]  = wilson_data.w_data[0][0][0].pa_head[i] + 
	((desc_count[0][i] * sizeof(DMA_InjDescriptor_t)) >> 4);
      wilson_data.w_data[0][0][1].pa_head[i]  = wilson_data.w_data[0][0][0].pa_tail[i] +
	(sizeof(DMA_InjDescriptor_t) >> 4);
      wilson_data.w_data[0][0][1].pa_tail[i]  = wilson_data.w_data[0][0][1].pa_head[i] +
	((desc_count[0][i] * sizeof(DMA_InjDescriptor_t)) >> 4);
	
      wilson_data.w_data[0][1][0].pa_head[i]  = wilson_data.w_data[0][0][1].pa_tail[i] +
	(sizeof(DMA_InjDescriptor_t) >> 4);
      wilson_data.w_data[0][1][0].pa_tail[i]  = wilson_data.w_data[0][1][0].pa_head[i] +
	((desc_count[1][i] * sizeof(DMA_InjDescriptor_t)) >> 4);
      wilson_data.w_data[0][1][1].pa_head[i]  = wilson_data.w_data[0][1][0].pa_tail[i] +
	(sizeof(DMA_InjDescriptor_t) >> 4);
      wilson_data.w_data[0][1][1].pa_tail[i]  = wilson_data.w_data[0][1][1].pa_head[i] +
	((desc_count[1][i] * sizeof(DMA_InjDescriptor_t)) >> 4);
	
      if (wilson_data.w_data[0][1][1].pa_tail[i]  == injection_fg.fifos[fifo_id[i]].dma_fifo.fifo_hw_ptr->pa_end)
	wilson_data.w_data[0][1][1].pa_tail[i]  = injection_fg.fifos[fifo_id[i]].dma_fifo.pa_start;
    }
  }
  MPI_Barrier(MPI_COMM_WORLD);
}

void a_dirac(double* in, double* out, int* oe);

// oe refers to outp, U
void bgp_wilson_hop(double* in, double* out, int* oe, int* dagger, int* single)
{
  unsigned c2 = wilson_data.mod2_ctr;

  uint32_t in_pa;
  int r = Kernel_VaTo4bitShiftedPa(in, 1, &in_pa);
  if (r)
    die("Cannot VaToPa");
  for (unsigned i=0; i<8; ++i)
  {
    wilson_data.w_data[*single][*oe][c2].inj_counter_hw[i].counter =
      wilson_data.w_data[*single][*oe][c2].fifo_size[i];
    wilson_data.w_data[*single][*oe][c2].inj_counter_hw[i].pa_base = in_pa;
  }

  _bgp_mbar();

  for (unsigned i=0; i<8; ++i)
    wilson_data.w_data[*single][*oe][c2].fifo_hw[i]->pa_head = wilson_data.w_data[*single][*oe][c2].pa_head[i];
  _bgp_mbar();
  for (unsigned i=0; i<8; ++i)
    wilson_data.w_data[*single][*oe][c2].fifo_hw[i]->pa_tail = wilson_data.w_data[*single][*oe][c2].pa_tail[i];
  _bgp_mbar();

  if (*dagger == 0)
    a_dirac(in, out, oe);
  if (*dagger == 1)
    a_dirac_dg(in, out, oe);

  for (unsigned i=0; i<8; ++i)
    wilson_data.w_data[*single][*oe][c2].rec_counter_hw[i].increment =
      wilson_data.w_data[*single][1-*oe][c2].fifo_size[i];
  _bgp_mbar();

// make sure fifo is empty so that input spinor may be overwritten

  for (unsigned i=0; i<8; ++i)
    while (wilson_data.w_data[*single][*oe][c2].inj_counter_hw[i].counter != 0);
}
