
/*******************************************************************************
*
* File msap.c
*
* Copyright (C) 2005 Martin Luescher, 2007 Bjoern Leder, 2011 Andrea Nobile
*
* This software is distributed under the terms of the GNU General Public
* License (GPL)
*
* Multiplicative alternating Schwarz procedure for the solution of the
* Wilson-Dirac equation D*psi=eta
*
* The externally accessible functions are
*
*   void msap(int nmr,int isr,int isx)
*     Application of one cycle of the multiplicative Schwarz procedure
*     to the global single-precision spinor field ps[isx], assuming the
*     associated residue is stored in the field ps[isr] and using nmr
*     minimal residual steps to solve the Dirac equation on the blocks.
*     The new approximate solution and residue are returned in the same
*     fields
*
*   void free_msap_bufs(void)
*     Frees the communication buffers allocated by msap
*
* Notes:
*
* The programs act on the GCR_BLOCKS block grid. It is taken for granted
* that the grid is allocated and that the gauge and sw fields on the grid
* are in the proper condition when msap() is called. The spinor fields
* b.s[k] (k=0,1,2) on the blocks are used as workspace
*
* All programs in this file act globally and should be called in parallel
* on all processes with the same arguments. The required communication
* buffers are allocated automatically if they are not already allocated.
* Even-odd preconditioning is used for the solution of the linear systems
* on the blocks
*
*******************************************************************************/

#define MSAP_C

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <float.h>
#include "mpi.h"
#include "su3.h"
#include "flags.h"
#include "start.h"
#include "linalg.h"
#include "block.h"
#include "sw_term.h"
#include "dirac.h"
#include "sap_gcr.h"
#include "global.h"



#ifdef __SSE__
#include <xmmintrin.h>
#endif

#ifdef __SSE2__
#include <emmintrin.h>
#endif

#ifdef __SSE3__
#include <pmmintrin.h>
#endif

static int nb,nbh,isw,init=0;
static int np,nmu[16],nbl[16],nsbf[16],nlbf[16],*imbf[16];
static int itags=0,tags[16];
static weyl *snd_buf[16],*loc_buf[16],*rcv_buf[16];
static block_t *b0;
static MPI_Request snd_req[16],rcv_req[16];


static void get_tags(void)
{
   int i;
   
   if (itags==0)
   {
      for (i=0;i<16;i++)
         tags[i]=mpi_permanent_tag();

      itags=1;
   }
}


static void set_nbf(void)
{
   int ifc,mu,*bs,ls[4];
   block_t *b,*bm;
   bndry_t *bb;

   np=(cpr[0]+cpr[1]+cpr[2]+cpr[3])&0x1;
   bs=(*b0).bs;

   ls[0]=L0;
   ls[1]=L1;
   ls[2]=L2;
   ls[3]=L3;

   for (ifc=0;ifc<16;ifc++)
   {
      mu=(ifc&0x7)/2;
      nmu[ifc]=cpr[mu]&0x1;
      nbl[ifc]=ls[mu]/bs[mu];

      nlbf[ifc]=0;
      nsbf[ifc]=0;

      if (ifc<8)
         b=b0;
      else
         b=b0+nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         bb=(*b).bb+(ifc&0x7);

         if ((*bb).ibn==0)
            nlbf[ifc]+=(*bb).vol;
         else
            nsbf[ifc]+=(*bb).vol;
      }
   }
}


static void alloc_weyl(void)
{
   int n,ifc;
   weyl *pw;

   n=0;

   for (ifc=0;ifc<16;ifc++)
      n+=(nlbf[ifc]+2*nsbf[ifc]);

   pw=amalloc(n*sizeof(weyl),ALIGN);
   error(pw==NULL,1,"alloc_weyl [msap.c]","Unable to allocate buffers");

   for (ifc=0;ifc<16;ifc++)
   {
      snd_buf[ifc]=pw;
      pw+=nsbf[ifc];
      loc_buf[ifc]=pw;
      pw+=nlbf[ifc];
      rcv_buf[ifc]=pw;
      pw+=nsbf[ifc];
   }
}


static void alloc_sbb(void)
{
   int ifc;
   weyl **ppw,*pwl,*pws;
   block_t *b,*bm;
   bndry_t *bb;

   ppw=amalloc(8*nb*sizeof(*ppw),3);
   error(ppw==NULL,1,"alloc_sbb [msap.c]",
         "Unable to allocate boundary fields");

   for (ifc=0;ifc<16;ifc++)
   {
      pwl=loc_buf[ifc];
      pws=snd_buf[ifc];

      if (ifc<8)
         b=b0;
      else
         b=b0+nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         bb=(*b).bb+(ifc&0x7);

         (*bb).ns=1;
         (*bb).s=ppw;

         if ((*bb).ibn==0)
         {
            *ppw=pwl;
            pwl+=(*bb).vol;
         }
         else
         {
            *ppw=pws;
            pws+=(*bb).vol;
         }

         ppw+=1;
      }
   }
}


static void assign_i2i(int n,int *ia,int *ib)
{
   int *im;

   im=ia+n;

   for (;ia<im;ia++,ib++)
      *ib=*ia;
}


static void alloc_imb(void)
{
   int n,ifc,*imb;
   block_t *b,*bm;
   bndry_t *bb;

   n=0;

   for (ifc=0;ifc<16;ifc++)
      n+=(nlbf[ifc]+nsbf[ifc]);

   imb=amalloc(n*sizeof(int),3);
   error(imb==NULL,1,"alloc_imb [msap.c]",
         "Unable to allocate index arrays");

   for (ifc=0;ifc<16;ifc++)
   {
      imbf[ifc]=imb;

      if (ifc<8)
         b=b0;
      else
         b=b0+nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         bb=(*b).bb+(ifc&0x7);

         if ((*bb).ibn==0)
         {
            assign_i2i((*bb).vol,(*bb).imb,imb);
            imb+=(*bb).vol;
         }
      }

      if (((ifc+8*nbl[ifc])&0xf)<8)
         b=b0;
      else
         b=b0+nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         bb=(*b).bb+(ifc&0x7);

         if ((*bb).ibn==1)
         {
            assign_i2i((*bb).vol,(*bb).imb,imb);
            imb+=(*bb).vol;
         }
      }
   }
}


static void set_req(void)
{
   int ifc,jfc,saddr,raddr,tag;

   get_tags();
   
   for (ifc=0;ifc<16;ifc++)
   {
      if (isw==1)
         jfc=(ifc+8)&0xf;
      else
         jfc=ifc;

      saddr=npr[(jfc&0x7)];
      raddr=npr[(jfc&0x7)^0x1];
      tag=tags[ifc];

      if (nsbf[jfc]>0)
      {
         MPI_Send_init((float*)(snd_buf[jfc]),12*nsbf[jfc],
                       MPI_FLOAT,saddr,tag,MPI_COMM_WORLD,&snd_req[jfc]);
         MPI_Recv_init((float*)(rcv_buf[jfc]),12*nsbf[jfc],
                       MPI_FLOAT,raddr,tag,MPI_COMM_WORLD,&rcv_req[jfc]);
      }
   }
}


static void alloc_msap_bufs(void)
{
   bndry_t *bb;

   b0=blk_list(GCR_BLOCKS,&nb,&isw);
   error(b0==NULL,1,"alloc_msap_bufs [msap.c]",
         "Block grid is not allocated");

   bb=(*b0).bb;
   error((bb==NULL)||((*bb).ns!=0),1,"alloc_msap_bufs [msap.c]",
          "Block boundary is not in proper condition");

   nbh=nb/2;
   set_nbf();
   alloc_weyl();
   alloc_sbb();
   alloc_imb();
   set_req();

   init=1;
   set_flags(ALLOCATED_MSAP_BUFS);
}


static void send_buf(int ifc,int eo)
{
   int io;
   
   if (np==eo)
   {
      io=(ifc^nmu[ifc]);

      if (nsbf[io]>0)
         MPI_Start(&snd_req[io]);
   }
   else
   {
      io=(ifc^nmu[ifc])^0x1;
      
      if (nsbf[io]>0)
         MPI_Start(&rcv_req[io]);
   }
}


static void wait_buf(int ifc,int eo)
{
   int io;
   MPI_Status stat_snd,stat_rcv;

   if (np==eo)
   {
      io=(ifc^nmu[ifc]);

      if (nsbf[io]>0)
         MPI_Wait(&snd_req[io],&stat_snd);      
   }
   else
   {
      io=(ifc^nmu[ifc])^0x1;
      
      if (nsbf[io]>0)
         MPI_Wait(&rcv_req[io],&stat_rcv);
   }
}

#if (defined SSE)
#include "sse.h"

static void upd_x(block_t *b,int isx)
{
   int *imb;
   spinor *psx,*rsx,*rsy,*rsr,*rsb,*rsm;

   imb=(*b).imb;
   psx=ps[isx][0];
   rsy=psx+(*imb);
   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      rsx=rsy;
      imb+=1;
      rsy=psx+(*imb);
      _prefetch_spinor(rsy);
      
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;
      
      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsx).c1.c1),
                            "m" ((*rsx).c1.c3),
                            "m" ((*rsx).c2.c2),
                            "m" ((*rsx).c3.c1),
                            "m" ((*rsx).c3.c3),
                            "m" ((*rsx).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsx).c1.c1),
                            "=m" ((*rsx).c1.c3),
                            "=m" ((*rsx).c2.c2),
                            "=m" ((*rsx).c3.c1),
                            "=m" ((*rsx).c3.c3),
                            "=m" ((*rsx).c4.c2));
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      rsx=rsy;
      imb+=1;
      rsy=psx+(*imb);
      _prefetch_spinor(rsy);

      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsr).c1.c1),
                            "m" ((*rsr).c1.c3),
                            "m" ((*rsr).c2.c2),
                            "m" ((*rsr).c3.c1),
                            "m" ((*rsr).c3.c3),
                            "m" ((*rsr).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsr+=4;
      _prefetch_spinor(rsr);
      rsr-=3;
      
      __asm__ __volatile__ ("subps %0, %%xmm0 \n\t"
                            "subps %1, %%xmm1 \n\t"
                            "subps %2, %%xmm2 \n\t"
                            "subps %3, %%xmm3 \n\t"
                            "subps %4, %%xmm4 \n\t"
                            "subps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsb).c1.c1),
                            "=m" ((*rsb).c1.c3),
                            "=m" ((*rsb).c2.c2),
                            "=m" ((*rsb).c3.c1),
                            "=m" ((*rsb).c3.c3),
                            "=m" ((*rsb).c4.c2));

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsx).c1.c1),
                            "m" ((*rsx).c1.c3),
                            "m" ((*rsx).c2.c2),
                            "m" ((*rsx).c3.c1),
                            "m" ((*rsx).c3.c3),
                            "m" ((*rsx).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsx).c1.c1),
                            "=m" ((*rsx).c1.c3),
                            "=m" ((*rsx).c2.c2),
                            "=m" ((*rsx).c3.c1),
                            "=m" ((*rsx).c3.c3),
                            "=m" ((*rsx).c4.c2));
   }
}


static void upd_x_ptr(block_t *b,spinor *psx)
{
   int *imb;
   spinor *rsx,*rsy,*rsr,*rsb,*rsm;

   imb=(*b).imb;
   
   rsy=psx+(*imb);
   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      rsx=rsy;
      imb+=1;
      rsy=psx+(*imb);
      _prefetch_spinor(rsy);
      
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;
      
      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsx).c1.c1),
                            "m" ((*rsx).c1.c3),
                            "m" ((*rsx).c2.c2),
                            "m" ((*rsx).c3.c1),
                            "m" ((*rsx).c3.c3),
                            "m" ((*rsx).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsx).c1.c1),
                            "=m" ((*rsx).c1.c3),
                            "=m" ((*rsx).c2.c2),
                            "=m" ((*rsx).c3.c1),
                            "=m" ((*rsx).c3.c3),
                            "=m" ((*rsx).c4.c2));
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      rsx=rsy;
      imb+=1;
      rsy=psx+(*imb);
      _prefetch_spinor(rsy);

      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsr).c1.c1),
                            "m" ((*rsr).c1.c3),
                            "m" ((*rsr).c2.c2),
                            "m" ((*rsr).c3.c1),
                            "m" ((*rsr).c3.c3),
                            "m" ((*rsr).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsr+=4;
      _prefetch_spinor(rsr);
      rsr-=3;
      
      __asm__ __volatile__ ("subps %0, %%xmm0 \n\t"
                            "subps %1, %%xmm1 \n\t"
                            "subps %2, %%xmm2 \n\t"
                            "subps %3, %%xmm3 \n\t"
                            "subps %4, %%xmm4 \n\t"
                            "subps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsb).c1.c1),
                            "=m" ((*rsb).c1.c3),
                            "=m" ((*rsb).c2.c2),
                            "=m" ((*rsb).c3.c1),
                            "=m" ((*rsb).c3.c3),
                            "=m" ((*rsb).c4.c2));

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsx).c1.c1),
                            "m" ((*rsx).c1.c3),
                            "m" ((*rsx).c2.c2),
                            "m" ((*rsx).c3.c1),
                            "m" ((*rsx).c3.c3),
                            "m" ((*rsx).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsx).c1.c1),
                            "=m" ((*rsx).c1.c3),
                            "=m" ((*rsx).c2.c2),
                            "=m" ((*rsx).c3.c1),
                            "=m" ((*rsx).c3.c3),
                            "=m" ((*rsx).c4.c2));
   }
}


static void upd_r(block_t *b,int isr)
{
   int *imb,*imm;
   spinor *psr,*rsr,*rsb,*rsm;

   imb=(*b).imb;
   psr=ps[isr][0];
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;

      rsr=psr+(*imb);
      imb+=1;
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsr).c1.c1),
                            "=m" ((*rsr).c1.c3),
                            "=m" ((*rsr).c2.c2),
                            "=m" ((*rsr).c3.c1),
                            "=m" ((*rsr).c3.c3),
                            "=m" ((*rsr).c4.c2));
   }

   __asm__ __volatile__ ("xorps %%xmm0, %%xmm0 \n\t"
                         "xorps %%xmm1, %%xmm1 \n\t"
                         "xorps %%xmm2, %%xmm2"
                         :
                         :
                         :
                         "xmm0", "xmm1", "xmm2");

   imm=imb+(*b).volh;

   for (;imb<imm;imb++)
   {
      rsr=psr+(*imb);

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm0, %3 \n\t"
                            "movaps %%xmm1, %4 \n\t"
                            "movaps %%xmm2, %5"
                            :
                            "=m" ((*rsr).c1.c1),
                            "=m" ((*rsr).c1.c3),
                            "=m" ((*rsr).c2.c2),
                            "=m" ((*rsr).c3.c1),
                            "=m" ((*rsr).c3.c3),
                            "=m" ((*rsr).c4.c2));
   }
}


static void upd_r_ptr(block_t *b,spinor *psr)
{
   int *imb,*imm;
   spinor *rsr,*rsb,*rsm;

   imb=(*b).imb;
 
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*rsb).c1.c1),
                            "m" ((*rsb).c1.c3),
                            "m" ((*rsb).c2.c2),
                            "m" ((*rsb).c3.c1),
                            "m" ((*rsb).c3.c3),
                            "m" ((*rsb).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rsb+=4;
      _prefetch_spinor(rsb);
      rsb-=3;

      rsr=psr+(*imb);
      imb+=1;
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*rsr).c1.c1),
                            "=m" ((*rsr).c1.c3),
                            "=m" ((*rsr).c2.c2),
                            "=m" ((*rsr).c3.c1),
                            "=m" ((*rsr).c3.c3),
                            "=m" ((*rsr).c4.c2));
   }

   __asm__ __volatile__ ("xorps %%xmm0, %%xmm0 \n\t"
                         "xorps %%xmm1, %%xmm1 \n\t"
                         "xorps %%xmm2, %%xmm2"
                         :
                         :
                         :
                         "xmm0", "xmm1", "xmm2");

   imm=imb+(*b).volh;

   for (;imb<imm;imb++)
   {
      rsr=psr+(*imb);

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm0, %3 \n\t"
                            "movaps %%xmm1, %4 \n\t"
                            "movaps %%xmm2, %5"
                            :
                            "=m" ((*rsr).c1.c1),
                            "=m" ((*rsr).c1.c3),
                            "=m" ((*rsr).c2.c2),
                            "=m" ((*rsr).c3.c1),
                            "=m" ((*rsr).c3.c3),
                            "=m" ((*rsr).c4.c2));
   }
}

#elif (defined DH)
#include "dh.h"

static void upd_x(block_t *b,int isx)
{
   int *imb;
   spinor *psx,*rsx,*rsr,*rsb,*rsm;

#pragma disjoint(*rsx,*rsb,*rsr)
   __alignx(16,rsx);
   __alignx(16,rsb);
   __alignx(16,rsr);

   _dh_vector_declare(x1);
   _dh_vector_declare(x2);
   _dh_vector_declareu(b1);
   _dh_vector_declareu(b2);

   imb=(*b).imb;
   psx=ps[isx][0];
   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsx=psx+(*imb);
      imb+=1;

      _dhs_vector_load((*rsx).c1,x1);
      _dhs_vector_loadu((*rsb).c1,b1);
      _dhs_vector_load((*rsx).c2,x2);
      _dhs_vector_loadu((*rsb).c2,b2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c1,x1);
      _dhs_vector_store((*rsx).c2,x2);

      _dhs_vector_load((*rsx).c3,x1);
      _dhs_vector_loadu((*rsb).c3,b1);
      _dhs_vector_load((*rsx).c4,x2);
      _dhs_vector_loadu((*rsb).c4,b2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c3,x1);
      _dhs_vector_store((*rsx).c4,x2);
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      _dhs_vector_load((*rsr).c1,x1);
      _dhs_vector_load((*rsb).c1,b1);
      _dhs_vector_load((*rsr).c2,x2);
      _dhs_vector_load((*rsb).c2,b2);

      _dh_vector_sub(b1,x1,b1);
      _dh_vector_sub(b2,x2,b2);

      _dhs_vector_storeu((*rsb).c1,b1);
      _dhs_vector_storeu((*rsb).c2,b2);

      rsx=psx+(*imb);
      imb+=1;

      _dhs_vector_load((*rsx).c1,x1);
      _dhs_vector_load((*rsx).c2,x2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c1,x1);
      _dhs_vector_store((*rsx).c2,x2);


      _dhs_vector_load((*rsr).c3,x1);
      _dhs_vector_load((*rsb).c3,b1);
      _dhs_vector_load((*rsr).c4,x2);
      _dhs_vector_load((*rsb).c4,b2);

      rsr+=1;

      _dh_vector_sub(b1,x1,b1);
      _dh_vector_sub(b2,x2,b2);

      _dhs_vector_storeu((*rsb).c3,b1);
      _dhs_vector_storeu((*rsb).c4,b2);

      _dhs_vector_load((*rsx).c3,x1);
      _dhs_vector_load((*rsx).c4,x2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c3,x1);
      _dhs_vector_store((*rsx).c4,x2);
   }
}


static void upd_x_ptr(block_t *b,spinor *psx)
{
   int *imb;
   spinor *rsx,*rsr,*rsb,*rsm;

#pragma disjoint(*rsx,*rsb,*rsr)
   __alignx(16,rsx);
   __alignx(16,rsb);
   __alignx(16,rsr);

   _dh_vector_declare(x1);
   _dh_vector_declare(x2);
   _dh_vector_declareu(b1);
   _dh_vector_declareu(b2);

   imb=(*b).imb;
   
   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsx=psx+(*imb);
      imb+=1;

      _dhs_vector_load((*rsx).c1,x1);
      _dhs_vector_loadu((*rsb).c1,b1);
      _dhs_vector_load((*rsx).c2,x2);
      _dhs_vector_loadu((*rsb).c2,b2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c1,x1);
      _dhs_vector_store((*rsx).c2,x2);

      _dhs_vector_load((*rsx).c3,x1);
      _dhs_vector_loadu((*rsb).c3,b1);
      _dhs_vector_load((*rsx).c4,x2);
      _dhs_vector_loadu((*rsb).c4,b2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c3,x1);
      _dhs_vector_store((*rsx).c4,x2);
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      _dhs_vector_load((*rsr).c1,x1);
      _dhs_vector_load((*rsb).c1,b1);
      _dhs_vector_load((*rsr).c2,x2);
      _dhs_vector_load((*rsb).c2,b2);

      _dh_vector_sub(b1,x1,b1);
      _dh_vector_sub(b2,x2,b2);

      _dhs_vector_storeu((*rsb).c1,b1);
      _dhs_vector_storeu((*rsb).c2,b2);

      rsx=psx+(*imb);
      imb+=1;

      _dhs_vector_load((*rsx).c1,x1);
      _dhs_vector_load((*rsx).c2,x2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c1,x1);
      _dhs_vector_store((*rsx).c2,x2);


      _dhs_vector_load((*rsr).c3,x1);
      _dhs_vector_load((*rsb).c3,b1);
      _dhs_vector_load((*rsr).c4,x2);
      _dhs_vector_load((*rsb).c4,b2);

      rsr+=1;

      _dh_vector_sub(b1,x1,b1);
      _dh_vector_sub(b2,x2,b2);

      _dhs_vector_storeu((*rsb).c3,b1);
      _dhs_vector_storeu((*rsb).c4,b2);

      _dhs_vector_load((*rsx).c3,x1);
      _dhs_vector_load((*rsx).c4,x2);

      _dh_vector_add_assign2(x1,x1,b1);
      _dh_vector_add_assign2(x2,x2,b2);

      _dhs_vector_store((*rsx).c3,x1);
      _dhs_vector_store((*rsx).c4,x2);
   }
}


static void upd_r(block_t *b,int isr)
{
   int *imb;
   spinor *psr,*rsr,*rsb,*rsm;

#pragma disjoint(*rsb,*rsr)
   __alignx(16,rsb);
   __alignx(16,rsr);

   _dh_vector_declare1(1);
   _dh_vector_declare1(2);

   psr=ps[isr][0];

   imb=(*b).imb;
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;

      _dhs_vector_load((*rsb).c1,1);
      _dhs_vector_load((*rsb).c2,2);
      _dhs_vector_store((*rsr).c1,1);
      _dhs_vector_store((*rsr).c2,2);

      _dhs_vector_load((*rsb).c3,1);
      _dhs_vector_load((*rsb).c4,2);
      _dhs_vector_store((*rsr).c3,1);
      _dhs_vector_store((*rsr).c4,2);
   }

   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      __stfps(&(*rsr).c1.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c1.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c1.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c3.re,  __cmplx(0.0, 0.0));
   }
}



static void upd_r_ptr(block_t *b, spinor *psr)
{
   int *imb;
   spinor *psr,*rsr,*rsb,*rsm;

#pragma disjoint(*rsb,*rsr)
   __alignx(16,rsb);
   __alignx(16,rsr);

   _dh_vector_declare1(1);
   _dh_vector_declare1(2);



   imb=(*b).imb;
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;

      _dhs_vector_load((*rsb).c1,1);
      _dhs_vector_load((*rsb).c2,2);
      _dhs_vector_store((*rsr).c1,1);
      _dhs_vector_store((*rsr).c2,2);

      _dhs_vector_load((*rsb).c3,1);
      _dhs_vector_load((*rsb).c4,2);
      _dhs_vector_store((*rsr).c3,1);
      _dhs_vector_store((*rsr).c4,2);
   }

   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      __stfps(&(*rsr).c1.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c1.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c1.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c2.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c3.c3.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c1.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c2.re,  __cmplx(0.0, 0.0));
      __stfps(&(*rsr).c4.c3.re,  __cmplx(0.0, 0.0));
   }
}

#else

static const spinor s0={{{0.0f}}};

static void upd_x(block_t *b,int isx)
{
   int *imb;
   spinor *psx,*rsx,*rsr,*rsb,*rsm;

   imb=(*b).imb;
   psx=ps[isx][0];
   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsx=psx+(*imb);
      imb+=1;

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      _vector_sub((*rsb).c1,(*rsr).c1,(*rsb).c1);
      _vector_sub((*rsb).c2,(*rsr).c2,(*rsb).c2);
      _vector_sub((*rsb).c3,(*rsr).c3,(*rsb).c3);
      _vector_sub((*rsb).c4,(*rsr).c4,(*rsb).c4);

      rsr+=1;
      rsx=psx+(*imb);
      imb+=1;

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);
   }
}

static void upd_x_ptr(block_t *b,spinor *psx)
{
   int *imb;
   spinor *rsx,*rsr,*rsb,*rsm;

   imb=(*b).imb;

   rsb=(*b).s[0];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsx=psx+(*imb);
      imb+=1;

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);
   }

   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      _vector_sub((*rsb).c1,(*rsr).c1,(*rsb).c1);
      _vector_sub((*rsb).c2,(*rsr).c2,(*rsb).c2);
      _vector_sub((*rsb).c3,(*rsr).c3,(*rsb).c3);
      _vector_sub((*rsb).c4,(*rsr).c4,(*rsb).c4);

      rsr+=1;
      rsx=psx+(*imb);
      imb+=1;

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);
   }
}


static void upd_r(block_t *b,int isr)
{
   int *imb;
   spinor *psr,*rsr,*rsb,*rsm;

   psr=ps[isr][0];

   imb=(*b).imb;
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      *rsr=*rsb;
   }

   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      *rsr=s0;
   }
}


static void upd_r_ptr(block_t *b, spinor *psr)
{
   int *imb;
   spinor *rsr,*rsb,*rsm;



   imb=(*b).imb;
   rsb=(*b).s[1];
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      *rsr=*rsb;
   }

   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      *rsr=s0;
   }
}

#endif

void msap(int nmr,int isr,int isx)
{
   int ic,is,ifc,ifc0,ifcm,jfc,kfc;
   int vol,volh;
   spinor *ps0o,*ps1e,*ps1o;
   block_t *b,*bm;

   if (init==0)
      alloc_msap_bufs();

   vol=(*b0).vol;
   volh=(*b0).volh;
   ps0o=(*b0).s[0]+volh;
   ps1e=(*b0).s[1];
   ps1o=(*b0).s[1]+volh;

   for (ic=0;ic<2;ic++)
   {
      is=(isw+ic)&0x1;
      b=b0+is*nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         assign_s2sb(b,ALL_PTS,isr,1);
         mulg5(vol,ps1e);
         apply_sw(volh,(*b).sw+vol,ps1o,ps1o);
         Qeo_blk(b,1,1);
         mulg5(volh,ps1e);

         blk_mres(b,nmr);

         Qoe_blk(b,0,0);
         apply_sw(volh,(*b).sw+vol,ps0o,ps0o);
         upd_x(b,isx);
         upd_r(b,isr);
         Qbnds(b,0,0);
      }

      ifc0=is*8;
      ifcm=ifc0+8;

      send_buf(ifc0,0);
      wait_buf(ifc0,0);
      send_buf(ifc0,1);
      wait_buf(ifc0,1);
      
      for (ifc=ifc0;ifc<ifcm;ifc++)
      {
         jfc=ifc+1;
         
         if (jfc<ifcm)
         {
            send_buf(jfc,0);
            wait_buf(jfc,0);
            send_buf(jfc,1);
         }

         kfc=(ifc^nmu[ifc])^0x1;
         mulg5_sub_assign_w2s[kfc&0x7](imbf[kfc],nlbf[kfc]+nsbf[kfc],
                                       loc_buf[kfc],ps[isr][0]);

         if (jfc<ifcm)
            wait_buf(jfc,1);
      }
   }
}



void msap_ptr(int nmr, spinor *psr, spinor *psx)
{
   int ic,is,ifc,ifc0,ifcm,jfc,kfc;
   int vol,volh;
   spinor *ps0o,*ps1e,*ps1o;
   block_t *b,*bm;

   if (init==0)
      alloc_msap_bufs();

   vol=(*b0).vol;
   volh=(*b0).volh;
   ps0o=(*b0).s[0]+volh;
   ps1e=(*b0).s[1];
   ps1o=(*b0).s[1]+volh;

   for (ic=0;ic<2;ic++)
   {
      is=(isw+ic)&0x1;
      b=b0+is*nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         assign_s2sb_ptr(b,ALL_PTS,psr,1);
         mulg5(vol,ps1e);
         apply_sw(volh,(*b).sw+vol,ps1o,ps1o);
         Qeo_blk(b,1,1);
         mulg5(volh,ps1e);

         blk_mres(b,nmr);

         Qoe_blk(b,0,0);
         apply_sw(volh,(*b).sw+vol,ps0o,ps0o);
         upd_x_ptr(b,psx);
         upd_r_ptr(b,psr);
         Qbnds(b,0,0);
      }

      ifc0=is*8;
      ifcm=ifc0+8;

      send_buf(ifc0,0);
      wait_buf(ifc0,0);
      send_buf(ifc0,1);
      wait_buf(ifc0,1);
      
      for (ifc=ifc0;ifc<ifcm;ifc++)
      {
         jfc=ifc+1;
         
         if (jfc<ifcm)
         {
            send_buf(jfc,0);
            wait_buf(jfc,0);
            send_buf(jfc,1);
         }

         kfc=(ifc^nmu[ifc])^0x1;
         mulg5_sub_assign_w2s[kfc&0x7](imbf[kfc],nlbf[kfc]+nsbf[kfc],
                                       loc_buf[kfc],psr);

         if (jfc<ifcm)
            wait_buf(jfc,1);
      }
   }
}


void free_msap_bufs(void)
{
   int n,ifc;
   bndry_t *bb;

   if (init==0)
      return;

   for (ifc=0;ifc<16;ifc++)
   {
      if (nsbf[ifc]>0)
      {
         MPI_Request_free(&snd_req[ifc]);
         MPI_Request_free(&rcv_req[ifc]);
      }
   }

   afree(imbf[0]);

   for (n=0;n<nb;n++)
   {
      bb=(b0[n]).bb;
      
      if (n==0)
         afree((*bb).s);

      for (ifc=0;ifc<8;ifc++)
      {
         (*bb).s=NULL;
         (*bb).ns=0;
         bb+=1;
      }
   }

   afree(snd_buf[0]);   
   init=0;
   set_flags(FREED_MSAP_BUFS);
}

#ifdef SSE
static inline __attribute__((always_inline)) void sse_vaddass(spinor *pl, spinor rrs)
{


     float *ff;
     ff = (float*)pl;
     
     __m128 av, bv, cv, dv, ev, fv;
     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_add_ps(av, dv);
     bv = _mm_add_ps(bv, ev);
     cv = _mm_add_ps(cv, fv);
     
     ff = (float*)pl;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);

     ff+=12;


     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     ff+=12;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_add_ps(av, dv);
     bv = _mm_add_ps(bv, ev);
     cv = _mm_add_ps(cv, fv);
     
     ff = (float*)pl;
     
     ff+=12;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);
}

#endif



static void upd_x_gs(block_t *b, spinor *psx)
{
   int *imb;
   spinor *rsx,*rsb,*rsm;

   imb=(*b).imb;

   rsb=(*b).s[0];
   //rsm=rsb+(*b).volh;
   rsm=rsb+(*b).vol;

   for (;rsb<rsm;rsb++)
   {
      rsx=psx+(*imb);
      imb+=1;
      
      #ifdef SSE
      sse_vaddass(rsx, *rsb);
      #else

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);

      #endif
   }
   /*
   rsr=(*b).s[1]+(*b).volh;
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      _vector_sub((*rsb).c1,(*rsr).c1,(*rsb).c1);
      _vector_sub((*rsb).c2,(*rsr).c2,(*rsb).c2);
      _vector_sub((*rsb).c3,(*rsr).c3,(*rsb).c3);
      _vector_sub((*rsb).c4,(*rsr).c4,(*rsb).c4);

      rsr+=1;
      rsx=psx+(*imb);
      imb+=1;

      _vector_add_assign((*rsx).c1,(*rsb).c1);
      _vector_add_assign((*rsx).c2,(*rsb).c2);
      _vector_add_assign((*rsx).c3,(*rsb).c3);
      _vector_add_assign((*rsx).c4,(*rsb).c4);
   }
   */
}


static void upd_r_gs(block_t *b, spinor *psr)
{
   int *imb;
   spinor *rsr,*rsb,*rsm;

   

   imb=(*b).imb;
   rsb=(*b).s[1];
   //rsm=rsb+(*b).volh;

   rsm=rsb+(*b).vol;
   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      #ifdef SSE
      __m128 a, bv, c, d, e, f;
      float *fi = (float*) rsb;
      float *fo = (float*) rsr;
      a = _mm_load_ps(fi);
      bv = _mm_load_ps(fi+4);
      c = _mm_load_ps(fi+8);
      d = _mm_load_ps(fi+12);
      e = _mm_load_ps(fi+16);
      f = _mm_load_ps(fi+20);
      
      _mm_store_ps(fo,   a);
      _mm_store_ps(fo+4, bv);
      _mm_store_ps(fo+8, c);
      _mm_store_ps(fo+12, d);
      _mm_store_ps(fo+16, e);
      _mm_store_ps(fo+20, f);



      #else
      *rsr=*rsb;
      #endif
   }

   /*
   rsm=rsb+(*b).volh;

   for (;rsb<rsm;rsb++)
   {
      rsr=psr+(*imb);
      imb+=1;
      *rsr=s0;
   }
   */
}

void msap_gs_ptr(int nmr,spinor *psr,spinor *psx)
{
   int ic,is,ifc,ifc0,ifcm,jfc,kfc;
  
   block_t *b,*bm;
   int i;
   if (init==0)
      alloc_msap_bufs();

  
  

   for (ic=0;ic<2;ic++)
   {
      is=(isw+ic)&0x1;
      //is = !is;
      b=b0+is*nbh;
      bm=b+nbh;

      for (;b<bm;b++)
      {
         assign_s2sb_ptr(b,ALL_PTS,psr,1);
	 //printf("GS!!!\n");
	 set_s2zero(b->vol,b->s[0]);
	 
	 for (i=0;i<nmr;i++) {
	 }
	 // gs_blk(b, 1 , 0);
	 
	 //printf("OUT GS!!!\n");
	 /*
	 mulg5(vol,ps1e);
         apply_sw(volh,(*b).sw+vol,ps1o,ps1o);
         Qeo_blk(b,1,1);
         mulg5(volh,ps1e);

         blk_mres(b,nmr);

         Qoe_blk(b,0,0);
         apply_sw(volh,(*b).sw+vol,ps0o,ps0o);
         */

	 upd_x_gs(b,psx);
         upd_r_gs(b,psr);
         Qbnds(b,0,0);
      }

      ifc0=is*8;
      ifcm=ifc0+8;

      send_buf(ifc0,0);
      wait_buf(ifc0,0);
      send_buf(ifc0,1);
      wait_buf(ifc0,1);
      
      for (ifc=ifc0;ifc<ifcm;ifc++)
      {
         jfc=ifc+1;
         
         if (jfc<ifcm)
         {
            send_buf(jfc,0);
            wait_buf(jfc,0);
            send_buf(jfc,1);
         }

         kfc=(ifc^nmu[ifc])^0x1;
         mulg5_sub_assign_w2s[kfc&0x7](imbf[kfc],nlbf[kfc]+nsbf[kfc],
                                       loc_buf[kfc],psr);

         if (jfc<ifcm)
            wait_buf(jfc,1);
      }
   }
}

