
/*******************************************************************************
*
* File linalg.c
*
* Copyright (C) 2005, 2007 Martin Luescher, 2007 Bjoern Leder
*
* This software is distributed under the terms of the GNU General Public
* License (GPL)
*
* Generic linear algebra routines for single-precision Dirac fields
*
* The externally accessible functions are
*
*   complex spinor_prod(int vol,int icom,spinor *pk,spinor *pl)
*     Computes the scalar product of the fields pk[] and pl[]
*
*   float spinor_prod_re(int vol,int icom,spinor *pk,spinor *pl)
*     Computes the real part of the scalar product of the fields
*     pk[] and pl[]
*
*   float norm_square(int vol,int icom,spinor *pk)
*     Computes the square of the norm of the field pk[]
*
*   void mulc_spinor_add(int vol,spinor *pk,spinor *pl,complex z)
*     Replaces pk[] by pk[]+z*pl[]
*
*   void project(int vol,int icom,spinor *pk,spinor *pl)
*     Replaces ps_k by pk[]-(pl,pk)*pl[]
*
*   float normalize(int vol,int icom,spinor *pk)
*     Replaces pk[] by pk[]/||pk|| and returns the norm ||pk||
*
*   void rotate(int vol,int n,spinor **ppk,complex v[])
*     Replaces p_k[] by sum_j p_j[]*v[n*j+k] where 0<=k,j<n and p_k=ppk[k]
*
*   void mulg5(int vol,spinor *pk)
*     Multiplies the field pk[] with gamma_5
*
*   void mulmg5(int vol,spinor *pk)
*     Multiplies the field pk[] with -gamma_5
*
*   void mulg5_spinor_sub(int vol,spinor *pk,spinor *pl)
*     Replaces pk[] by pk[]-gamma_5*pl[]
*
* Notes:
*
* All these programs operate on arrays of spinor fields whose base address
* is passed through the arguments. The length of the arrays is specified 
* by the parameter vol. Scalar products etc. are globally summed if the 
* parameter icom is equal to 1. In this case the calculated values are
* guaranteed to be exactly the same on all processes
*
* The base address of the spinor fields must be a multiple of 16 if SSE/DH
* instructions are to be used
*
*******************************************************************************/

#define LINALG_C

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include "mpi.h"
#include "su3.h"
#include "start.h"
#include "global.h"

static int nrot=0,ifail=0;
static spinor *psi,*chi;


static void alloc_wrotate(int n)
{
   if (nrot>0)
      afree(psi);
   
   psi=amalloc(2*n*sizeof(*psi),ALIGN);

   if (error_loc(psi==NULL,1,"alloc_wrotate [linalg.c]",
                 "Unable to allocate workspace")!=0)
   {
      nrot=0;
      ifail=1;      
      chi=NULL;
   }
   else
   {
      nrot=n;
      chi=psi+n;
      set_s2zero(2*n,psi);
   }
}


#if (defined SSE)
#include "sse.h"

static sse_float tmp1,tmp2;


complex spinor_prod(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;
   double x,y;
   complex_dble v,w;
   spinor *pkm;
  
   x=0.0;
   y=0.0;
   pkm=pk+vol;

   for (;pk<pkm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("mulps %0, %%xmm0 \n\t"
                            "mulps %1, %%xmm1 \n\t"
                            "mulps %2, %%xmm2 \n\t"
                            "mulps %3, %%xmm3 \n\t"
                            "mulps %4, %%xmm4 \n\t"
                            "mulps %5, %%xmm5"
                            :
                            :
                            "m" ((*pl).c1.c1),
                            "m" ((*pl).c1.c3),
                            "m" ((*pl).c2.c2),
                            "m" ((*pl).c3.c1),
                            "m" ((*pl).c3.c3),
                            "m" ((*pl).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=4;
      
      __asm__ __volatile__ ("addps %%xmm0, %%xmm1 \n\t"
                            "addps %%xmm2, %%xmm3 \n\t"
                            "addps %%xmm4, %%xmm5 \n\t"
                            "addps %%xmm1, %%xmm3 \n\t"
                            "addps %%xmm3, %%xmm5 \n\t"
                            :
                            :
                            :
                            "xmm1", "xmm3", "xmm5");

      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm6"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");

      pl+=4;
      _prefetch_spinor(pl);
      pl-=4;
   
      __asm__ __volatile__ ("shufps $0xb1, %%xmm0, %%xmm0 \n\t"
                            "shufps $0xb1, %%xmm1, %%xmm1 \n\t"
                            "shufps $0xb1, %%xmm2, %%xmm2 \n\t"
                            "shufps $0xb1, %%xmm3, %%xmm3 \n\t"
                            "shufps $0xb1, %%xmm4, %%xmm4 \n\t"
                            "shufps $0xb1, %%xmm6, %%xmm6"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");
      
      __asm__ __volatile__ ("mulps %0, %%xmm0 \n\t"
                            "mulps %1, %%xmm1 \n\t"
                            "mulps %2, %%xmm2 \n\t"
                            "mulps %3, %%xmm3 \n\t"
                            "mulps %4, %%xmm4 \n\t"
                            "mulps %5, %%xmm6"
                            :
                            :
                            "m" ((*pl).c1.c1),
                            "m" ((*pl).c1.c3),
                            "m" ((*pl).c2.c2),
                            "m" ((*pl).c3.c1),
                            "m" ((*pl).c3.c3),
                            "m" ((*pl).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");
      
      __asm__ __volatile__ ("addps %%xmm0, %%xmm1 \n\t"
                            "addps %%xmm2, %%xmm3 \n\t"
                            "addps %%xmm4, %%xmm6 \n\t"
                            "addps %%xmm1, %%xmm3 \n\t"
                            "addps %%xmm3, %%xmm6 \n\t"
                            "movaps %%xmm5, %0 \n\t"
                            "movaps %%xmm6, %1"                            
                            :
                            "=m" (tmp1),
                            "=m" (tmp2)
                            :
                            :
                            "xmm1", "xmm3", "xmm6");

      x+=(double)(tmp1.c1+tmp1.c2+tmp1.c3+tmp1.c4);      
      y+=(double)(tmp2.c2-tmp2.c1+tmp2.c4-tmp2.c3);
      pk+=1;
      pl+=1;
   }


   if ((icom==1)&&(NPROC>1))
   {
      v.re=x;
      v.im=y;

      MPI_Reduce(&v.re,&w.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&w.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);     

      z.re=(float)(w.re);
      z.im=(float)(w.im);
   }
   else
   {
      z.re=(float)(x);
      z.im=(float)(y);
   }
   
   return z;  
}


float spinor_prod_re(int vol,int icom,spinor *pk,spinor *pl)
{
   double x,y;
   spinor *pkm;
  
   x=0.0;
   pkm=pk+vol;
   
   for (;pk<pkm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=3;
      
      __asm__ __volatile__ ("mulps %0, %%xmm0 \n\t"
                            "mulps %1, %%xmm1 \n\t"
                            "mulps %2, %%xmm2 \n\t"
                            "mulps %3, %%xmm3 \n\t"
                            "mulps %4, %%xmm4 \n\t"
                            "mulps %5, %%xmm5"
                            :
                            :
                            "m" ((*pl).c1.c1),
                            "m" ((*pl).c1.c3),
                            "m" ((*pl).c2.c2),
                            "m" ((*pl).c3.c1),
                            "m" ((*pl).c3.c3),
                            "m" ((*pl).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pl+=4;
      _prefetch_spinor(pl);
      pl-=3;
      
      __asm__ __volatile__ ("addps %%xmm0, %%xmm1 \n\t"
                            "addps %%xmm2, %%xmm3 \n\t"
                            "addps %%xmm4, %%xmm5 \n\t"
                            "addps %%xmm1, %%xmm3 \n\t"
                            "addps %%xmm3, %%xmm5 \n\t"
                            "movaps %%xmm5, %0"
                            :
                            "=m" (tmp1)
                            :
                            :
                            "xmm1", "xmm3", "xmm5");

      x+=(double)(tmp1.c1+tmp1.c2+tmp1.c3+tmp1.c4);
   }

   if ((icom==1)&&(NPROC>1))
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
   else
      return (float)(x);
}


float norm_square(int vol,int icom,spinor *pk)
{
   double x,y;
   spinor *pkm;
  
   x=0.0;
   pkm=pk+vol;
   
   for (;pk<pkm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=3;
      
      __asm__ __volatile__ ("mulps %%xmm0, %%xmm0 \n\t"
                            "mulps %%xmm1, %%xmm1 \n\t"
                            "mulps %%xmm2, %%xmm2 \n\t"
                            "mulps %%xmm3, %%xmm3 \n\t"
                            "mulps %%xmm4, %%xmm4 \n\t"
                            "mulps %%xmm5, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("addps %%xmm0, %%xmm1 \n\t"
                            "addps %%xmm2, %%xmm3 \n\t"
                            "addps %%xmm4, %%xmm5 \n\t"
                            "addps %%xmm1, %%xmm3 \n\t"
                            "addps %%xmm3, %%xmm5 \n\t"
                            "movaps %%xmm5, %0"
                            :
                            "=m" (tmp1)
                            :
                            :
                            "xmm1", "xmm3", "xmm5");

      x+=(double)(tmp1.c1+tmp1.c2+tmp1.c3+tmp1.c4);
   }

   if ((icom==1)&&(NPROC>1))
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
   else
      return (float)(x);
}


void mulc_spinor_add(int vol,spinor *pk,spinor *pl,complex z)
{
   spinor *pkm;

   _sse_load_cmplx(z);
   pkm=pk+vol;
   
   for (;pk<pkm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %%xmm0, %%xmm3 \n\t"
                            "movaps %%xmm1, %%xmm4 \n\t"
                            "movaps %%xmm2, %%xmm5 \n\t"
                            :
                            :
                            "m" ((*pl).c1.c1),
                            "m" ((*pl).c1.c3),
                            "m" ((*pl).c2.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pl+=4;
      _prefetch_spinor(pl);
      pl-=4;
      
      __asm__ __volatile__ ("mulps %%xmm6, %%xmm0 \n\t"
                            "mulps %%xmm6, %%xmm1 \n\t"
                            "mulps %%xmm6, %%xmm2 \n\t"
                            "shufps $0xb1, %%xmm3, %%xmm3 \n\t"
                            "shufps $0xb1, %%xmm4, %%xmm4 \n\t"
                            "shufps $0xb1, %%xmm5, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "mulps %%xmm7, %%xmm3 \n\t"
                            "mulps %%xmm7, %%xmm4 \n\t"
                            "mulps %%xmm7, %%xmm5 \n\t"
                            "addps %%xmm0, %%xmm3 \n\t"
                            "addps %%xmm1, %%xmm4 \n\t"
                            "addps %%xmm2, %%xmm5"                            
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %3, %%xmm0 \n\t"
                            "movaps %4, %%xmm1 \n\t"
                            "movaps %5, %%xmm2 \n\t"
                            "movaps %%xmm3, %0 \n\t"
                            "movaps %%xmm4, %1 \n\t"
                            "movaps %%xmm5, %2 \n\t"
                            "movaps %%xmm0, %%xmm3 \n\t"
                            "movaps %%xmm1, %%xmm4 \n\t"
                            "movaps %%xmm2, %%xmm5"
                            :
                            "=m" ((*pk).c1.c1),
                            "=m" ((*pk).c1.c3),
                            "=m" ((*pk).c2.c2)
                            :
                            "m" ((*pl).c3.c1),
                            "m" ((*pl).c3.c3),
                            "m" ((*pl).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=4;
      
      __asm__ __volatile__ ("mulps %%xmm6, %%xmm0 \n\t"
                            "mulps %%xmm6, %%xmm1 \n\t"
                            "mulps %%xmm6, %%xmm2 \n\t"
                            "shufps $0xb1, %%xmm3, %%xmm3 \n\t"
                            "shufps $0xb1, %%xmm4, %%xmm4 \n\t"
                            "shufps $0xb1, %%xmm5, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "mulps %%xmm7, %%xmm3 \n\t"
                            "mulps %%xmm7, %%xmm4 \n\t"
                            "mulps %%xmm7, %%xmm5 \n\t"
                            "addps %%xmm0, %%xmm3 \n\t"
                            "addps %%xmm1, %%xmm4 \n\t"
                            "addps %%xmm2, %%xmm5"                            
                            :
                            :
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movaps %%xmm3, %0 \n\t"
                            "movaps %%xmm4, %1 \n\t"
                            "movaps %%xmm5, %2 \n\t"
                            :
                            "=m" ((*pk).c3.c1),
                            "=m" ((*pk).c3.c3),
                            "=m" ((*pk).c4.c2));

      pk+=1;
      pl+=1;
   }
}


void project(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;

   z=spinor_prod(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add(vol,pk,pl,z);   
}


float normalize(int vol,int icom,spinor *pk)
{
   float r,ri;
   spinor *pkm;   

   r=norm_square(vol,icom,pk);
   r=(float)(sqrt((double)(r)));

   if (error_loc(r==0.0f,1,"normalize [linalg.c]",
                 "Vector has vanishing norm")!=0)
      return 0.0f;

   ri=1.0f/r;

   __asm__ __volatile__ ("movss %0, %%xmm6 \n\t"
                         "shufps $0x0, %%xmm6, %%xmm6 \n\t"
                         "movaps %%xmm6, %%xmm7"
                         :
                         :
                         "m" (ri)
                         :
                         "xmm6", "xmm7");

   pkm=pk+vol;
   
   for (;pk<pkm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=4;      
      
      __asm__ __volatile__ ("mulps %%xmm6, %%xmm0 \n\t"
                            "mulps %%xmm7, %%xmm1 \n\t"
                            "mulps %%xmm6, %%xmm2 \n\t"
                            "mulps %%xmm7, %%xmm3 \n\t"
                            "mulps %%xmm6, %%xmm4 \n\t"
                            "mulps %%xmm7, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*pk).c1.c1),
                            "=m" ((*pk).c1.c3),
                            "=m" ((*pk).c2.c2),
                            "=m" ((*pk).c3.c1),
                            "=m" ((*pk).c3.c3),
                            "=m" ((*pk).c4.c2));

      pk+=1;
   }

   return r;
}


void rotate(int vol,int n,spinor **ppk,complex v[])
{
   int k,j,ix;
   complex *z;
   sse_vector *pk,*pj;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {   
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)
            chi[k]=*(ppk[k]+ix);
      
         pk=(sse_vector*)(psi);
      
         for (k=0;k<n;k++)  
         {
            pj=(sse_vector*)(chi);
            z=&v[k];

            _sse_vector_load(*pj);
            _sse_load_cmplx(*z);
            _sse_mulc();
     
            for (j=1;j<n;j++)
            {
               pj+=2;
               z+=n;

               _sse_vector_load(*pj);
               _sse_load_cmplx(*z);
               _sse_add_mulc_re();
               _sse_vector_load(*pj);
               _sse_add_mulc_im();
            }

            _sse_vector_store_up(*pk);
            pk+=1;

            pj=(sse_vector*)(chi)+1;
            z=&v[k];

            _sse_vector_load(*pj);
            _sse_load_cmplx(*z);
            _sse_mulc();
     
            for (j=1;j<n;j++)
            {
               pj+=2;
               z+=n;

               _sse_vector_load(*pj);
               _sse_load_cmplx(*z);
               _sse_add_mulc_re();
               _sse_vector_load(*pj);
               _sse_add_mulc_im();
            }

            _sse_vector_store_up(*pk);
            pk+=1;         
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psi[k];
      }
   }
}


void mulg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;
   
   __asm__ __volatile__ ("movaps %0, %%xmm5 \n\t"
                         "movaps %%xmm5, %%xmm6 \n\t"
                         "movaps %%xmm5, %%xmm7"
                         :
                         :
                         "m" (_sse_sgn)
                         :
                         "xmm5", "xmm6", "xmm7");                         

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;   
   
   for (;rpk<rpm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c2)
                            :
                            "xmm0", "xmm1", "xmm2");

      rpk+=8;
      _prefetch_weyl(rpk);
      rpk-=8;          
      
      __asm__ __volatile__ ("mulps %%xmm5, %%xmm0 \n\t"
                            "mulps %%xmm6, %%xmm1 \n\t"
                            "mulps %%xmm7, %%xmm2"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2\t"
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c3),
                            "=m" ((*rpk).c2.c2));

      rpk+=2;
   }
}


void mulmg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;
   
   __asm__ __volatile__ ("movaps %0, %%xmm5 \n\t"
                         "movaps %%xmm5, %%xmm6 \n\t"
                         "movaps %%xmm5, %%xmm7"
                         :
                         :
                         "m" (_sse_sgn)
                         :
                         "xmm5", "xmm6", "xmm7");                         

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c2)
                            :
                            "xmm0", "xmm1", "xmm2");

      rpk+=8;
      _prefetch_weyl(rpk);
      rpk-=8;          
      
      __asm__ __volatile__ ("mulps %%xmm5, %%xmm0 \n\t"
                            "mulps %%xmm6, %%xmm1 \n\t"
                            "mulps %%xmm7, %%xmm2"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2");

      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2\t"
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c3),
                            "=m" ((*rpk).c2.c2));

      rpk+=2;          
   }
}


void mulg5_spinor_sub(int vol,spinor *pk,spinor *pl)
{
   spinor *pkm;

   pkm=pk+vol;

   for (;pk<pkm;pk++)
   {
      __asm__ __volatile__ ("movaps %0, %%xmm0 \n\t"
                            "movaps %1, %%xmm1 \n\t"
                            "movaps %2, %%xmm2 \n\t"
                            "movaps %3, %%xmm3 \n\t"
                            "movaps %4, %%xmm4 \n\t"
                            "movaps %5, %%xmm5"
                            :
                            :
                            "m" ((*pk).c1.c1),
                            "m" ((*pk).c1.c3),
                            "m" ((*pk).c2.c2),
                            "m" ((*pk).c3.c1),
                            "m" ((*pk).c3.c3),
                            "m" ((*pk).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pk+=4;
      _prefetch_spinor(pk);
      pk-=4;
      
      __asm__ __volatile__ ("subps %0, %%xmm0 \n\t"
                            "subps %1, %%xmm1 \n\t"
                            "subps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" ((*pl).c1.c1),
                            "m" ((*pl).c1.c3),
                            "m" ((*pl).c2.c2),
                            "m" ((*pl).c3.c1),
                            "m" ((*pl).c3.c3),
                            "m" ((*pl).c4.c2)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      pl+=4;
      _prefetch_spinor(pl);
      pl-=3;
      
      __asm__ __volatile__ ("movaps %%xmm0, %0 \n\t"
                            "movaps %%xmm1, %1 \n\t"
                            "movaps %%xmm2, %2 \n\t"
                            "movaps %%xmm3, %3 \n\t"
                            "movaps %%xmm4, %4 \n\t"
                            "movaps %%xmm5, %5"
                            :
                            "=m" ((*pk).c1.c1),
                            "=m" ((*pk).c1.c3),
                            "=m" ((*pk).c2.c2),
                            "=m" ((*pk).c3.c1),
                            "=m" ((*pk).c3.c3),
                            "=m" ((*pk).c4.c2));
   }
}

#elif (defined DH)
#include "dh.h"

complex spinor_prod(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;
   complex_dble v,w;
   spinor *rpk,*rpl;

   __alignx(16,rpk);
   __alignx(16,rpl);

   _dh_vector_declare2(1);
   _dh_vector_declare2(2);
   double _Complex sp;
   sp = __cmplx(0.0, 0.0);

   rpk=pk;
   rpl=pl;

   for (;rpk<(pk+vol);)
   {
      _dhs_vector_load2((*rpk).c1,(*rpl).c1,1);
      _dhs_vector_load2((*rpk).c2,(*rpl).c2,2);

      _dh_vector_prod2(1,2);
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);


      _dhs_vector_load2((*rpk).c3,(*rpl).c3,1);
      _dhs_vector_load2((*rpk).c4,(*rpl).c4,2);

      _dh_vector_prod2(1,2); 
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);

      rpk+=1;
      rpl+=1;
   }

   if ((icom!=1)||(NPROC==1))
   {
      __stfps(&z.re,  sp);
   }
   else
   {
      __stfpd(&v.re,  sp);

      MPI_Reduce(&v.re,&w.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&w.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);     

      z.re=(float)(w.re);
      z.im=(float)(w.im);
   }

   return z;
}


float spinor_prod_re(int vol,int icom,spinor *pk,spinor *pl)
{
   double x,y;
   spinor *rpk,*rpl;

   __alignx(16,rpk);
   __alignx(16,rpl);

   _dh_vector_declare2(1);
   _dh_vector_declare2(2);
   double _Complex sp;
   sp = __cmplx(0.0, 0.0);

   rpk=pk;
   rpl=pl;

   for (;rpk<(pk+vol);)
   {
      _dhs_vector_load2((*rpk).c1,(*rpl).c1,1);
      _dhs_vector_load2((*rpk).c2,(*rpl).c2,2);
      _dh_vector_prod2(1,2);
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);


      _dhs_vector_load2((*rpk).c3,(*rpl).c3,1);
      _dhs_vector_load2((*rpk).c4,(*rpl).c4,2);

      _dh_vector_prod2(1,2);        
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);


      rpk+=1;
      rpl+=1;
   }

   x=__creal(sp);

   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}


float norm_square(int vol,int icom,spinor *pk)
{
   double x,y;
   spinor *rpk;

   __alignx(16,rpk);
   
   _dh_vector_declare(1);
   _dh_vector_declare(2);
   double _Complex sp;
   sp = __cmplx(0.0, 0.0);
   
   rpk=pk;
 
   for (;rpk<(pk+vol);)
   {
      _dhs_vector_load((*rpk).c1,1);
      _dhs_vector_load((*rpk).c2,2);
      
      _dh_vector_square2(1,2);
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);
      
      
      _dhs_vector_load((*rpk).c3,1);
      _dhs_vector_load((*rpk).c4,2);
      
      _dh_vector_square2(1,2);         
      _dh_vector_addu(sp,1);
      _dh_vector_addu(sp,2);
      
      rpk+=1;
   }

   x=__creal(sp);
   
   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}


void mulc_spinor_add(int vol,spinor *pk,spinor *pl,complex z)
{
   spinor *rpk,*rpl;

   __alignx(16,rpk);
   __alignx(16,rpl);
   
   _dh_vector_declare2(1); 
   _dh_vector_declare2(2); 
   double _Complex dcz;
   
   rpk=pk;
   rpl=pl;
   
   dcz = __lfps(&z.re); 
   for (;rpk<(pk+vol);)
   {
      _dhs_vector_load2((*rpk).c1,(*rpl).c1,1);
      _dhs_vector_load2((*rpk).c2,(*rpl).c2,2);
      
      _dh_vector_mulc_assign2(dcz,1,2);
      
      _dhs_vector_storeu((*rpk).c1,1);
      _dhs_vector_storeu((*rpk).c2,2);
      
      
      _dhs_vector_load2((*rpk).c3,(*rpl).c3,1);
      _dhs_vector_load2((*rpk).c4,(*rpl).c4,2);
      
      _dh_vector_mulc_assign2(dcz,1,2);
      
      _dhs_vector_storeu((*rpk).c3,1);
      _dhs_vector_storeu((*rpk).c4,2);

      rpk+=1;
      rpl+=1;
   }
}


void project(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;

   z=spinor_prod(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add(vol,pk,pl,z);   
}


float normalize(int vol,int icom,spinor *pk)
{
   float r;
   spinor *rpk;
   complex z;

   __alignx(16,rpk);
  
   _dh_vector_declare2(1); 
   double _Complex dcz;
   
   r=norm_square(vol,icom,pk);
   r=(float)(sqrt((double)(r)));

   if (error_loc(r==0.0f,1,
       "normalize [linalg.c]","Vector has vanishing norm")!=0)
      return 0.0f;

   z.re=1.0f/r;
   rpk=pk;
   
   dcz = __lfps(&z.re); 
   for (;rpk<(pk+vol);)
   {
      _dhs_vector_load2((*rpk).c1,(*rpk).c2,1);
      _dh_vector_mul(dcz,1);
      _dhs_vector_storeu((*rpk).c1,1);
      _dhs_vector_storet((*rpk).c2,1);
      
      _dhs_vector_load2((*rpk).c3,(*rpk).c4,1);    
      _dh_vector_mul(dcz,1);
      _dhs_vector_storeu((*rpk).c3,1);
      _dhs_vector_storet((*rpk).c4,1);
      
      rpk+=1;
   }

   return r;
}


void rotate(int vol,int n,spinor **ppk,complex v[])
{
   int k,j,ix;
   complex *z;
   spinor *pk,*pj;

   __alignx(16,pk);
   __alignx(16,pj);
   
   _dh_vector_declare(1); 
   _dh_vector_declare(2); 
   _dh_vector_declare(3); 
   _dh_vector_declare(4); 
   double _Complex dcz;
   
   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {     
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psi[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            dcz = __lfps(&(*z).re);
            _dhs_vector_load((*pj).c1,1);        
            _dhs_vector_load((*pj).c2,2);        
            _dhs_vector_load((*pj).c3,3);        
            _dhs_vector_load((*pj).c4,4);        
                  
            _dh_vector_mulc2(dcz,1,2);
            _dh_vector_mulc2(dcz,3,4);
              
            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;

               dcz = __lfps(&(*z).re);
               _dhs_vector_load((*pj).c1,1);
               _dhs_vector_load((*pj).c2,2);
               _dhs_vector_load((*pj).c3,3);
               _dhs_vector_load((*pj).c4,4);
            
               _dh_vector_mulc_add2(dcz,1,2);
               _dh_vector_mulc_add2(dcz,3,4);         
            }
            _dhs_vector_storeu((*pk).c1,1);
            _dhs_vector_storeu((*pk).c2,2);
            _dhs_vector_storeu((*pk).c3,3);
            _dhs_vector_storeu((*pk).c4,4);
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psi[k];
      }
   }
}


void mulg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   __alignx(16,rpk);
   
   _dh_vector_declareu(1);
   _dh_vector_declareu(2);
   
   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;rpk+=2)
   {
      _dhs_vector_load((*rpk).c1,1);
      _dhs_vector_load((*rpk).c2,2);
      
      _dh_vector_neg(1);
      _dh_vector_neg(2);
      
      _dhs_vector_storeu((*rpk).c1,1);
      _dhs_vector_storeu((*rpk).c2,2);    
   }
}


void mulmg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   __alignx(16,rpk);
   
   _dh_vector_declareu(1);
   _dh_vector_declareu(2);
   
   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;rpk+=2)
   {
      _dhs_vector_load((*rpk).c1,1);
      _dhs_vector_load((*rpk).c2,2);
      
      _dh_vector_neg(1);
      _dh_vector_neg(2);
      
      _dhs_vector_storeu((*rpk).c1,1);
      _dhs_vector_storeu((*rpk).c2,2);    
   }
}


void mulg5_spinor_sub(int vol,spinor *pk,spinor *pl)
{
   spinor *pkm;

   __alignx(16,pk);
   
   _dh_vector_declare1(1);
   _dh_vector_declare1(2);
   _dh_vector_declare1(3);
   _dh_vector_declare1(4);
   
   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      _dhs_vector_load((*pk).c1,1);
      _dhs_vector_load((*pk).c2,2);
      _dhs_vector_load((*pl).c1,3);
      _dhs_vector_load((*pl).c2,4);
      
      _dh_vector_sub_assign(1,3);
      _dh_vector_sub_assign(2,4);
      
      _dhs_vector_store((*pk).c1,1);
      _dhs_vector_store((*pk).c2,2);
      
      _dhs_vector_load((*pk).c3,1);
      _dhs_vector_load((*pk).c4,2);
      _dhs_vector_load((*pl).c3,3);
      _dhs_vector_load((*pl).c4,4);
      
      _dh_vector_add_assign(1,3);
      _dh_vector_add_assign(2,4);
     
      _dhs_vector_store((*pk).c3,1);
      _dhs_vector_store((*pk).c4,2);
      
      pl+=1;
   }
}

#elif (defined QPX)
#include "qpx.h"

complex spinor_prod(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;
   double x,y;
   complex_dble v,w;
   spinor *rpk,*rpl;

   vector4double prod;
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;
   vector4double vec_l_0, vec_l_1, vec_l_2, vec_l_3, vec_l_4, vec_l_5;

   prod = vec_splats( 0.0 );
   x=0.0;
   y=0.0;
   rpk=pk;
   rpl=pl;

   for (;rpk<(pk+vol);)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*rpk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*rpk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*rpk).c4.c2.re );

      vec_l_0 = VEC_LOAD( 0l, &(*rpl).c1.c1.re );
      vec_l_1 = VEC_LOAD( 0l, &(*rpl).c1.c3.re );
      vec_l_2 = VEC_LOAD( 0l, &(*rpl).c2.c2.re );
      vec_l_3 = VEC_LOAD( 0l, &(*rpl).c3.c1.re );
      vec_l_4 = VEC_LOAD( 0l, &(*rpl).c3.c3.re );
      vec_l_5 = VEC_LOAD( 0l, &(*rpl).c4.c2.re );

      prod = vec_xmadd( vec_k_0, vec_l_0, prod );
      prod = vec_xxcpnmadd( vec_l_0, vec_k_0, prod );

      prod = vec_xmadd( vec_k_1, vec_l_1, prod );
      prod = vec_xxcpnmadd( vec_l_1, vec_k_1, prod );

      prod = vec_xmadd( vec_k_2, vec_l_2, prod );
      prod = vec_xxcpnmadd( vec_l_2, vec_k_2, prod );

      prod = vec_xmadd( vec_k_3, vec_l_3, prod );
      prod = vec_xxcpnmadd( vec_l_3, vec_k_3, prod );

      prod = vec_xmadd( vec_k_4, vec_l_4, prod );
      prod = vec_xxcpnmadd( vec_l_4, vec_k_4, prod );

      prod = vec_xmadd( vec_k_5, vec_l_5, prod );
      prod = vec_xxcpnmadd( vec_l_5, vec_k_5, prod );

      rpk+=1;
      rpl+=1;
   }

   if ((icom!=1)||(NPROC==1))
   {
      z.re=(float)(x);
      z.im=(float)(y);

      z.re=(float)(prod[0]+prod[2]);
      z.im=(float)(prod[1]+prod[3]);
   }
   else
   {
      v.re=x;
      v.im=y;

      v.re=(float)(prod[0]+prod[2]);
      v.im=(float)(prod[1]+prod[3]);

      MPI_Reduce(&v.re,&w.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&w.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);     

      z.re=(float)(w.re);
      z.im=(float)(w.im);
   }
   
   return z;
}

float spinor_prod_re(int vol,int icom,spinor *pk,spinor *pl)
{
   double x,y;
   spinor *rpk,*rpl;
 
   vector4double prod = (vector4double){0.,0.,0.,0.};
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;
   vector4double vec_l_0, vec_l_1, vec_l_2, vec_l_3, vec_l_4, vec_l_5;

   x=0.0;
   rpk=pk;
   rpl=pl;
   
   for (;rpk<(pk+vol);)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*rpk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*rpk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*rpk).c4.c2.re );

      vec_l_0 = VEC_LOAD( 0l, &(*rpl).c1.c1.re );
      vec_l_1 = VEC_LOAD( 0l, &(*rpl).c1.c3.re );
      vec_l_2 = VEC_LOAD( 0l, &(*rpl).c2.c2.re );
      vec_l_3 = VEC_LOAD( 0l, &(*rpl).c3.c1.re );
      vec_l_4 = VEC_LOAD( 0l, &(*rpl).c3.c3.re );
      vec_l_5 = VEC_LOAD( 0l, &(*rpl).c4.c2.re );

      prod = vec_madd( vec_k_0, vec_l_0, prod );
      prod = vec_madd( vec_k_1, vec_l_1, prod );
      prod = vec_madd( vec_k_2, vec_l_2, prod );
      prod = vec_madd( vec_k_3, vec_l_3, prod );
      prod = vec_madd( vec_k_4, vec_l_4, prod );
      prod = vec_madd( vec_k_5, vec_l_5, prod );

      rpk+=1;
      rpl+=1;
   }

   x = prod[0] + prod[1] + prod[2] + prod[3];

   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}

float norm_square(int vol,int icom,spinor *pk)
{
   double x,y;
   spinor *rpk;

   vector4double norm;
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;

   norm = vec_splats( 0. );
   x=0.0;
   rpk=pk;
 
   for (;rpk<(pk+vol);)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*rpk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*rpk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*rpk).c4.c2.re );

      norm = vec_madd( vec_k_0, vec_k_0, norm );
      norm = vec_madd( vec_k_1, vec_k_1, norm );
      norm = vec_madd( vec_k_2, vec_k_2, norm );
      norm = vec_madd( vec_k_3, vec_k_3, norm );
      norm = vec_madd( vec_k_4, vec_k_4, norm );
      norm = vec_madd( vec_k_5, vec_k_5, norm );

      rpk+=1;
   }

   x = norm[0] + norm[1] + norm[2] + norm[3];

   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}

void mulc_spinor_add(int vol,spinor *pk,spinor *pl,complex z)
{
   spinor *rpk,*rpl;

   rpk=pk;
   rpl=pl;
   
   vector4double fact = (vector4double){ z.re, z.im, z.re, z.im };
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;
   vector4double vec_l_0, vec_l_1, vec_l_2, vec_l_3, vec_l_4, vec_l_5;

   for (;rpk<(pk+vol);)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*rpk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*rpk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*rpk).c4.c2.re );

      vec_l_0 = VEC_LOAD( 0l, &(*rpl).c1.c1.re );
      vec_l_1 = VEC_LOAD( 0l, &(*rpl).c1.c3.re );
      vec_l_2 = VEC_LOAD( 0l, &(*rpl).c2.c2.re );
      vec_l_3 = VEC_LOAD( 0l, &(*rpl).c3.c1.re );
      vec_l_4 = VEC_LOAD( 0l, &(*rpl).c3.c3.re );
      vec_l_5 = VEC_LOAD( 0l, &(*rpl).c4.c2.re );

      vec_k_0 = vec_xmadd( vec_l_0, fact, vec_k_0 );
      vec_k_1 = vec_xmadd( vec_l_1, fact, vec_k_1 );
      vec_k_2 = vec_xmadd( vec_l_2, fact, vec_k_2 );
      vec_k_3 = vec_xmadd( vec_l_3, fact, vec_k_3 );
      vec_k_4 = vec_xmadd( vec_l_4, fact, vec_k_4 );
      vec_k_5 = vec_xmadd( vec_l_5, fact, vec_k_5 );

      vec_k_0 = vec_xxnpmadd( fact, vec_l_0, vec_k_0 );
      vec_k_1 = vec_xxnpmadd( fact, vec_l_1, vec_k_1 );
      vec_k_2 = vec_xxnpmadd( fact, vec_l_2, vec_k_2 );
      vec_k_3 = vec_xxnpmadd( fact, vec_l_3, vec_k_3 );
      vec_k_4 = vec_xxnpmadd( fact, vec_l_4, vec_k_4 );
      vec_k_5 = vec_xxnpmadd( fact, vec_l_5, vec_k_5 );

      VEC_STORE( vec_k_0, 0l, &(*rpk).c1.c1.re );
      VEC_STORE( vec_k_1, 0l, &(*rpk).c1.c3.re );
      VEC_STORE( vec_k_2, 0l, &(*rpk).c2.c2.re );
      VEC_STORE( vec_k_3, 0l, &(*rpk).c3.c1.re );
      VEC_STORE( vec_k_4, 0l, &(*rpk).c3.c3.re );
      VEC_STORE( vec_k_5, 0l, &(*rpk).c4.c2.re );

      rpk+=1;
      rpl+=1;
   }
}

/* ------------------------- NO OPTIMIZATION -------------------------------- */

void project(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;

   z=spinor_prod(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add(vol,pk,pl,z);   
}

float normalize(int vol,int icom,spinor *pk)
{
   float r,ri;
   spinor *rpk;

   vector4double fact;
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;

   r=norm_square(vol,icom,pk);
   r=(float)(sqrt((double)(r)));

   if (error_loc(r==0.0f,1,
       "normalize [linalg.c]","Vector has vanishing norm")!=0)
      return 0.0f;

   ri=1.0f/r;
   fact = vec_splats( ri );
   rpk=pk;
   
   for (;rpk<(pk+vol);)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*rpk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*rpk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*rpk).c4.c2.re );

      vec_k_0 = vec_mul( vec_k_0, fact );
      vec_k_1 = vec_mul( vec_k_1, fact );
      vec_k_2 = vec_mul( vec_k_2, fact );
      vec_k_3 = vec_mul( vec_k_3, fact );
      vec_k_4 = vec_mul( vec_k_4, fact );
      vec_k_5 = vec_mul( vec_k_5, fact );

      VEC_STORE( vec_k_0, 0l, &(*rpk).c1.c1.re );
      VEC_STORE( vec_k_1, 0l, &(*rpk).c1.c3.re );
      VEC_STORE( vec_k_2, 0l, &(*rpk).c2.c2.re );
      VEC_STORE( vec_k_3, 0l, &(*rpk).c3.c1.re );
      VEC_STORE( vec_k_4, 0l, &(*rpk).c3.c3.re );
      VEC_STORE( vec_k_5, 0l, &(*rpk).c4.c2.re );

      rpk+=1;
   }

   return r;
}

void rotate(int vol,int n,spinor **ppk,complex v[])
{
   int k,j,ix;
   complex *z;
   spinor *pk,*pj;

   vector4double fact;
   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;
   vector4double vec_l_0, vec_l_1, vec_l_2, vec_l_3, vec_l_4, vec_l_5;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {     
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psi[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            fact = VEC_LOAD2( 0l, &(*z).re );

            vec_l_0 = VEC_LOAD( 0l, &(*pj).c1.c1.re );
            vec_l_1 = VEC_LOAD( 0l, &(*pj).c1.c3.re );
            vec_l_2 = VEC_LOAD( 0l, &(*pj).c2.c2.re );
            vec_l_3 = VEC_LOAD( 0l, &(*pj).c3.c1.re );
            vec_l_4 = VEC_LOAD( 0l, &(*pj).c3.c3.re );
            vec_l_5 = VEC_LOAD( 0l, &(*pj).c4.c2.re );

            vec_k_0 = vec_xmul( vec_l_0, fact );
            vec_k_1 = vec_xmul( vec_l_1, fact );
            vec_k_2 = vec_xmul( vec_l_2, fact );
            vec_k_3 = vec_xmul( vec_l_3, fact );
            vec_k_4 = vec_xmul( vec_l_4, fact );
            vec_k_5 = vec_xmul( vec_l_5, fact );

            vec_k_0 = vec_xxnpmadd( fact, vec_l_0, vec_k_0 );
            vec_k_1 = vec_xxnpmadd( fact, vec_l_1, vec_k_1 );
            vec_k_2 = vec_xxnpmadd( fact, vec_l_2, vec_k_2 );
            vec_k_3 = vec_xxnpmadd( fact, vec_l_3, vec_k_3 );
            vec_k_4 = vec_xxnpmadd( fact, vec_l_4, vec_k_4 );
            vec_k_5 = vec_xxnpmadd( fact, vec_l_5, vec_k_5 );

            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;
            
               fact = vec_splats( (*z).re );
               fact[1] = fact[3] = (*z).im;

               vec_l_0 = VEC_LOAD( 0l, &(*pj).c1.c1.re );
               vec_l_1 = VEC_LOAD( 0l, &(*pj).c1.c3.re );
               vec_l_2 = VEC_LOAD( 0l, &(*pj).c2.c2.re );
               vec_l_3 = VEC_LOAD( 0l, &(*pj).c3.c1.re );
               vec_l_4 = VEC_LOAD( 0l, &(*pj).c3.c3.re );
               vec_l_5 = VEC_LOAD( 0l, &(*pj).c4.c2.re );

               vec_k_0 = vec_xmadd( vec_l_0, fact, vec_k_0 );
               vec_k_1 = vec_xmadd( vec_l_1, fact, vec_k_1 );
               vec_k_2 = vec_xmadd( vec_l_2, fact, vec_k_2 );
               vec_k_3 = vec_xmadd( vec_l_3, fact, vec_k_3 );
               vec_k_4 = vec_xmadd( vec_l_4, fact, vec_k_4 );
               vec_k_5 = vec_xmadd( vec_l_5, fact, vec_k_5 );

               vec_k_0 = vec_xxnpmadd( fact, vec_l_0, vec_k_0 );
               vec_k_1 = vec_xxnpmadd( fact, vec_l_1, vec_k_1 );
               vec_k_2 = vec_xxnpmadd( fact, vec_l_2, vec_k_2 );
               vec_k_3 = vec_xxnpmadd( fact, vec_l_3, vec_k_3 );
               vec_k_4 = vec_xxnpmadd( fact, vec_l_4, vec_k_4 );
               vec_k_5 = vec_xxnpmadd( fact, vec_l_5, vec_k_5 );
            }

            VEC_STORE( vec_k_0, 0l, &(*pk).c1.c1.re );
            VEC_STORE( vec_k_1, 0l, &(*pk).c1.c3.re );
            VEC_STORE( vec_k_2, 0l, &(*pk).c2.c2.re );
            VEC_STORE( vec_k_3, 0l, &(*pk).c3.c1.re );
            VEC_STORE( vec_k_4, 0l, &(*pk).c3.c3.re );
            VEC_STORE( vec_k_5, 0l, &(*pk).c4.c2.re );
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psi[k];
      }
   }
}

void mulg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   vector4double weyl_k_0, weyl_k_1, weyl_k_2;

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;rpk+=2)
   {
      weyl_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      weyl_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      weyl_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );

      weyl_k_0 = vec_neg( weyl_k_0 );
      weyl_k_1 = vec_neg( weyl_k_1 );
      weyl_k_2 = vec_neg( weyl_k_2 );
      
      VEC_STORE( weyl_k_0, 0l, &(*rpk).c1.c1.re );
      VEC_STORE( weyl_k_1, 0l, &(*rpk).c1.c3.re );
      VEC_STORE( weyl_k_2, 0l, &(*rpk).c2.c2.re );
   }
}

void mulmg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   vector4double weyl_k_0, weyl_k_1, weyl_k_2;

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;rpk+=2)
   {
      weyl_k_0 = VEC_LOAD( 0l, &(*rpk).c1.c1.re );
      weyl_k_1 = VEC_LOAD( 0l, &(*rpk).c1.c3.re );
      weyl_k_2 = VEC_LOAD( 0l, &(*rpk).c2.c2.re );

      weyl_k_0 = vec_neg( weyl_k_0 );
      weyl_k_1 = vec_neg( weyl_k_1 );
      weyl_k_2 = vec_neg( weyl_k_2 );
      
      VEC_STORE( weyl_k_0, 0l, &(*rpk).c1.c1.re );
      VEC_STORE( weyl_k_1, 0l, &(*rpk).c1.c3.re );
      VEC_STORE( weyl_k_2, 0l, &(*rpk).c2.c2.re );
   }
}

void mulg5_spinor_sub(int vol,spinor *pk,spinor *pl)
{
   spinor *pkm;

   vector4double vec_k_0, vec_k_1, vec_k_2, vec_k_3, vec_k_4, vec_k_5;
   vector4double vec_l_0, vec_l_1, vec_l_2, vec_l_3, vec_l_4, vec_l_5;

   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      vec_k_0 = VEC_LOAD( 0l, &(*pk).c1.c1.re );
      vec_k_1 = VEC_LOAD( 0l, &(*pk).c1.c3.re );
      vec_k_2 = VEC_LOAD( 0l, &(*pk).c2.c2.re );
      vec_k_3 = VEC_LOAD( 0l, &(*pk).c3.c1.re );
      vec_k_4 = VEC_LOAD( 0l, &(*pk).c3.c3.re );
      vec_k_5 = VEC_LOAD( 0l, &(*pk).c4.c2.re );

      vec_l_0 = VEC_LOAD( 0l, &(*pl).c1.c1.re );
      vec_l_1 = VEC_LOAD( 0l, &(*pl).c1.c3.re );
      vec_l_2 = VEC_LOAD( 0l, &(*pl).c2.c2.re );
      vec_l_3 = VEC_LOAD( 0l, &(*pl).c3.c1.re );
      vec_l_4 = VEC_LOAD( 0l, &(*pl).c3.c3.re );
      vec_l_5 = VEC_LOAD( 0l, &(*pl).c4.c2.re );

      vec_k_0 = vec_sub( vec_k_0, vec_l_0 );
      vec_k_1 = vec_sub( vec_k_1, vec_l_1 );
      vec_k_2 = vec_sub( vec_k_2, vec_l_2 );
      vec_k_3 = vec_add( vec_k_3, vec_l_3 );
      vec_k_4 = vec_add( vec_k_4, vec_l_4 );
      vec_k_5 = vec_add( vec_k_5, vec_l_5 );

      VEC_STORE( vec_k_0, 0l, &(*pk).c1.c1.re );
      VEC_STORE( vec_k_1, 0l, &(*pk).c1.c3.re );
      VEC_STORE( vec_k_2, 0l, &(*pk).c2.c2.re );
      VEC_STORE( vec_k_3, 0l, &(*pk).c3.c1.re );
      VEC_STORE( vec_k_4, 0l, &(*pk).c3.c3.re );
      VEC_STORE( vec_k_5, 0l, &(*pk).c4.c2.re );

      pl+=1;
   }
}

#else

complex spinor_prod(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;
   double x,y;
   complex_dble v,w;
   spinor *rpk,*rpl;
  
   x=0.0;
   y=0.0;
   rpk=pk;
   rpl=pl;

   for (;rpk<(pk+vol);)
   {
      x+=(double)(_vector_prod_re((*rpk).c1,(*rpl).c1)+
            _vector_prod_re((*rpk).c2,(*rpl).c2)+
            _vector_prod_re((*rpk).c3,(*rpl).c3)+
            _vector_prod_re((*rpk).c4,(*rpl).c4));

      y+=(double)(_vector_prod_im((*rpk).c1,(*rpl).c1)+
            _vector_prod_im((*rpk).c2,(*rpl).c2)+
            _vector_prod_im((*rpk).c3,(*rpl).c3)+
            _vector_prod_im((*rpk).c4,(*rpl).c4));

      rpk+=1;
      rpl+=1;
   }

   if ((icom!=1)||(NPROC==1))
   {
      z.re=(float)(x);
      z.im=(float)(y);
   }
   else
   {
      v.re=x;
      v.im=y;

      MPI_Reduce(&v.re,&w.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&w.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);     

      z.re=(float)(w.re);
      z.im=(float)(w.im);
   }
   
   return z;  
}


float spinor_prod_re(int vol,int icom,spinor *pk,spinor *pl)
{
   double x,y;
   spinor *rpk,*rpl;
  
   x=0.0;
   rpk=pk;
   rpl=pl;
   
   for (;rpk<(pk+vol);)
   {
      x+=(double)(_vector_prod_re((*rpk).c1,(*rpl).c1)+
            _vector_prod_re((*rpk).c2,(*rpl).c2)+
            _vector_prod_re((*rpk).c3,(*rpl).c3)+
            _vector_prod_re((*rpk).c4,(*rpl).c4));

      rpk+=1;
      rpl+=1;
   }

   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}


float norm_square(int vol,int icom,spinor *pk)
{
   double x,y;
   spinor *rpk;

   x=0.0;
   rpk=pk;
 
   for (;rpk<(pk+vol);)
   {
      x+=(double)(_vector_prod_re((*rpk).c1,(*rpk).c1)+
            _vector_prod_re((*rpk).c2,(*rpk).c2)+
            _vector_prod_re((*rpk).c3,(*rpk).c3)+
            _vector_prod_re((*rpk).c4,(*rpk).c4));

      rpk+=1;
   }

   if ((icom!=1)||(NPROC==1))
      return (float)(x);
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return (float)(y);
   }
}


void mulc_spinor_add(int vol,spinor *pk,spinor *pl,complex z)
{
   spinor *rpk,*rpl;

   rpk=pk;
   rpl=pl;
   
   for (;rpk<(pk+vol);)
   {
      _vector_mulc_assign((*rpk).c1,z,(*rpl).c1);
      _vector_mulc_assign((*rpk).c2,z,(*rpl).c2);
      _vector_mulc_assign((*rpk).c3,z,(*rpl).c3);
      _vector_mulc_assign((*rpk).c4,z,(*rpl).c4);

      rpk+=1;
      rpl+=1;
   }
}


void project(int vol,int icom,spinor *pk,spinor *pl)
{
   complex z;

   z=spinor_prod(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add(vol,pk,pl,z);   
}


float normalize(int vol,int icom,spinor *pk)
{
   float r,ri;
   spinor *rpk;

   r=norm_square(vol,icom,pk);
   r=(float)(sqrt((double)(r)));

   if (error_loc(r==0.0f,1,
       "normalize [linalg.c]","Vector has vanishing norm")!=0)
      return 0.0f;

   ri=1.0f/r;
   rpk=pk;
   
   for (;rpk<(pk+vol);)
   {
      _vector_mul((*rpk).c1,ri,(*rpk).c1);
      _vector_mul((*rpk).c2,ri,(*rpk).c2);
      _vector_mul((*rpk).c3,ri,(*rpk).c3);
      _vector_mul((*rpk).c4,ri,(*rpk).c4);

      rpk+=1;
   }

   return r;
}


void rotate(int vol,int n,spinor **ppk,complex v[])
{
   int k,j,ix;
   complex *z;
   spinor *pk,*pj;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {     
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psi[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            _vector_mulc((*pk).c1,*z,(*pj).c1);
            _vector_mulc((*pk).c2,*z,(*pj).c2);
            _vector_mulc((*pk).c3,*z,(*pj).c3);
            _vector_mulc((*pk).c4,*z,(*pj).c4);
     
            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;

               _vector_mulc_assign((*pk).c1,*z,(*pj).c1);
               _vector_mulc_assign((*pk).c2,*z,(*pj).c2);
               _vector_mulc_assign((*pk).c3,*z,(*pj).c3);
               _vector_mulc_assign((*pk).c4,*z,(*pj).c4);            
            }
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psi[k];
      }
   }
}


void mulg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulmg5(int vol,spinor *pk)
{
   weyl *rpk,*rpm;

   rpk=(weyl*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulg5_spinor_sub(int vol,spinor *pk,spinor *pl)
{
   spinor *pkm;

   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      _vector_sub_assign((*pk).c1,(*pl).c1);
      _vector_sub_assign((*pk).c2,(*pl).c2);
      _vector_add_assign((*pk).c3,(*pl).c3);
      _vector_add_assign((*pk).c4,(*pl).c4);      

      pl+=1;
   }
}

#endif

//pl = pk + z*pl
void mulc_spinor_add2(int vol,spinor *pk,spinor *pl,complex z)
{
   spinor *rpk,*rpl;

   rpk=pk;
   rpl=pl;
   
   spinor tmp;
   
   

   for (;rpk<(pk+vol);)
   {
     
     tmp = *rpk;
     _vector_mulc_assign(tmp.c1,z,(*rpl).c1);
     _vector_mulc_assign(tmp.c2,z,(*rpl).c2);
     _vector_mulc_assign(tmp.c3,z,(*rpl).c3);
     _vector_mulc_assign(tmp.c4,z,(*rpl).c4);
     *rpl = tmp;
     rpk+=1;
     rpl+=1;
   }
}
