
/*******************************************************************************
*
* File linalg_dble.c
*
* Copyright (C) 2005, 2007 Martin Luescher, 2007 Bjoern Leder
*
* This software is distributed under the terms of the GNU General Public
* License (GPL)
*
* Generic linear algebra routines for double-precision Dirac fields
*
* The externally accessible functions are
*
*   complex_dble spinor_prod_dble(int vol,int icom,spinor_dble *pk,
*                                 spinor_dble *pl)
*     Computes the scalar product of the fields pk[] and pl[]
*
*   double spinor_prod_re_dble(int vol,int icom,spinor_dble *pk,
*                              spinor_dble *pl)
*     Computes the real part of the scalar product of the fields
*     pk[] and pl[]
*
*   double norm_square_dble(int vol,int icom,spinor_dble *pk)
*     Computes the square of the norm of the field pk[]
*
*   void mulc_spinor_add_dble(int vol,spinor_dble *pk,spinor_dble *pl,
*                             complex_dble z)
*     Replaces pk[] by pk[]+z*pl[]
*
*   void project_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
*     Replaces ps_k by pk[]-(pl,pk)*pl[]
*
*   double normalize_dble(int vol,int icom,spinor_dble *pk)
*     Replaces pk[] by pk[]/||pk|| and returns the norm ||pk||
*
*   void rotate_dble(int vol,int n,spinor_dble **ppk,complex_dble v[])
*     Replaces p_k[] by sum_j p_j[]*v[n*j+k] where 0<=k,j<n and p_k=ppk[k]
*
*   void mulg5_dble(int vol,spinor_dble *pk)
*     Multiplies the field pk[] with gamma_5
*
*   void mulmg5_dble(int vol,spinor_dble *pk)
*     Multiplies the field pk[] with -gamma_5
*
*   void mulg5_spinor_sub_dble(int vol,spinor_dble *pk,spinor_dble *pl)
*     Replaces pk[] by pk[]-gamma_5*pl[]
*
* Notes:
*
* All these programs operate on arrays of spinor fields whose base address
* is passed through the arguments. The length of the arrays is specified by
* the parameter vol. Scalar products etc. are globally summed if the 
* parameter icom is equal to 1. In this case the calculated values are
* guaranteed to be exactly the same on all processes
*
* The base address of the spinor fields must be a multiple of 16 if SSE2/DH
* instructions are to be used
*
*******************************************************************************/

#define LINALG_DBLE_C

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include "mpi.h"
#include "su3.h"
#include "start.h"
#include "global.h"

#define MAX_LEVELS 8
#define BLK_LENGTH 16

static int nrot=0,ifail=0;
static int cnt[MAX_LEVELS];
static double smx[MAX_LEVELS],smy[MAX_LEVELS];
static spinor_dble *psid,*chid;


static void alloc_wrotate(int n)
{
   if (nrot>0)
      afree(psid);
   
   psid=amalloc(2*n*sizeof(*psid),ALIGN);

   if (error_loc(psid==NULL,1,"alloc_wrotate [linalg_dble.c]",
                 "Unable to allocate workspace")!=0)
   {
      nrot=0;
      ifail=1;      
      chid=NULL;
   }
   else
   {
      nrot=n;
      chid=psid+n;
      set_sd2zero(2*n,psid);
   }
}


#if (defined SSE2)

#include "sse2.h"

static sse_double tmp1,tmp2;
static const sse_double sgn={-1.0,-1.0};

complex_dble spinor_prod_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   complex_dble w,z;
   weyl_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=(weyl_dble*)(pk);
   rpl=(weyl_dble*)(pl);
   rpmax=rpk;
   rptot=rpk+2*vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
      smy[n]=0.0;
   }

   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;
      y=0.0;

      for (;rpk<rpmax;)
      {
         __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                               "movapd %1, %%xmm1 \n\t"
                               "movapd %2, %%xmm2 \n\t"
                               "movapd %3, %%xmm3 \n\t"
                               "movapd %4, %%xmm4 \n\t"
                               "movapd %5, %%xmm5"
                               :
                               :
                               "m" ((*rpk).c1.c1),
                               "m" ((*rpk).c1.c2),
                               "m" ((*rpk).c1.c3),
                               "m" ((*rpk).c2.c1),
                               "m" ((*rpk).c2.c2),
                               "m" ((*rpk).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

         __asm__ __volatile__ ("mulpd %0, %%xmm0 \n\t"
                               "mulpd %1, %%xmm1 \n\t"
                               "mulpd %2, %%xmm2 \n\t"
                               "mulpd %3, %%xmm3 \n\t"
                               "mulpd %4, %%xmm4 \n\t"
                               "mulpd %5, %%xmm5"
                               :
                               :
                               "m" ((*rpl).c1.c1),
                               "m" ((*rpl).c1.c2),
                               "m" ((*rpl).c1.c3),
                               "m" ((*rpl).c2.c1),
                               "m" ((*rpl).c2.c2),
                               "m" ((*rpl).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

         rpk+=4;
         _prefetch_weyl_dble(rpk);
         rpk-=4;
         
         __asm__ __volatile__ ("addpd %%xmm0, %%xmm1 \n\t"
                               "addpd %%xmm2, %%xmm3 \n\t"
                               "addpd %%xmm4, %%xmm5 \n\t"
                               "addpd %%xmm1, %%xmm3 \n\t"
                               "addpd %%xmm3, %%xmm5"
                               :
                               :
                               :
                               "xmm1", "xmm3", "xmm5");
                               
         __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                               "movapd %1, %%xmm1 \n\t"
                               "movapd %2, %%xmm2 \n\t"
                               "movapd %3, %%xmm3 \n\t"
                               "movapd %4, %%xmm4 \n\t"
                               "movapd %5, %%xmm6"
                               :
                               :
                               "m" ((*rpk).c1.c1),
                               "m" ((*rpk).c1.c2),
                               "m" ((*rpk).c1.c3),
                               "m" ((*rpk).c2.c1),
                               "m" ((*rpk).c2.c2),
                               "m" ((*rpk).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");
         
         __asm__ __volatile__ ("shufpd $0x1, %%xmm0, %%xmm0 \n\t"
                               "shufpd $0x1, %%xmm1, %%xmm1 \n\t"
                               "shufpd $0x1, %%xmm2, %%xmm2 \n\t"
                               "shufpd $0x1, %%xmm3, %%xmm3 \n\t"
                               "shufpd $0x1, %%xmm4, %%xmm4 \n\t"
                               "shufpd $0x1, %%xmm6, %%xmm6"
                               :
                               :
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");

         __asm__ __volatile__ ("mulpd %0, %%xmm0 \n\t"
                               "mulpd %1, %%xmm1 \n\t"
                               "mulpd %2, %%xmm2 \n\t"
                               "mulpd %3, %%xmm3 \n\t"
                               "mulpd %4, %%xmm4 \n\t"
                               "mulpd %5, %%xmm6"
                               :
                               :
                               "m" ((*rpl).c1.c1),
                               "m" ((*rpl).c1.c2),
                               "m" ((*rpl).c1.c3),
                               "m" ((*rpl).c2.c1),
                               "m" ((*rpl).c2.c2),
                               "m" ((*rpl).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6");

         rpl+=4;
         _prefetch_weyl_dble(rpl);
         rpl-=3;
         
         __asm__ __volatile__ ("addpd %%xmm0, %%xmm1 \n\t"
                               "addpd %%xmm2, %%xmm3 \n\t"
                               "addpd %%xmm4, %%xmm6 \n\t"
                               "addpd %%xmm1, %%xmm3 \n\t"
                               "addpd %%xmm3, %%xmm6 \n\t"
                               "movapd %%xmm5, %0 \n\t"
                               "movapd %%xmm6, %1"
                               :
                               "=m" (tmp1),                               
                               "=m" (tmp2)
                               :
                               :
                               "xmm1", "xmm3", "xmm6");

         x+=(tmp1.c1+tmp1.c2);         
         y+=(tmp2.c2-tmp2.c1);
         rpk+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;
      smy[0]+=y;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];
         smy[n]+=smy[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
         smy[n-1]=0.0;
      }
   }

   x=0.0;
   y=0.0;

   for (n=0;n<MAX_LEVELS;n++)
   {
      x+=smx[n];
      y+=smy[n];
   }

   if ((icom==1)&&(NPROC>1))
   {
      w.re=x;
      w.im=y;      
      MPI_Reduce(&w.re,&z.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&z.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);  
   }
   else
   {
      z.re=x;
      z.im=y;
   }
   
   return z; 
}


double spinor_prod_re_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   weyl_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=(weyl_dble*)(pk);
   rpl=(weyl_dble*)(pl);
   rpmax=rpk;
   rptot=rpk+2*vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }

   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                               "movapd %1, %%xmm1 \n\t"
                               "movapd %2, %%xmm2 \n\t"
                               "movapd %3, %%xmm3 \n\t"
                               "movapd %4, %%xmm4 \n\t"
                               "movapd %5, %%xmm5"
                               :
                               :
                               "m" ((*rpk).c1.c1),
                               "m" ((*rpk).c1.c2),
                               "m" ((*rpk).c1.c3),
                               "m" ((*rpk).c2.c1),
                               "m" ((*rpk).c2.c2),
                               "m" ((*rpk).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

         rpk+=4;
         _prefetch_weyl_dble(rpk);
         rpk-=3;
         
         __asm__ __volatile__ ("mulpd %0, %%xmm0 \n\t"
                               "mulpd %1, %%xmm1 \n\t"
                               "mulpd %2, %%xmm2 \n\t"
                               "mulpd %3, %%xmm3 \n\t"
                               "mulpd %4, %%xmm4 \n\t"
                               "mulpd %5, %%xmm5"
                               :
                               :
                               "m" ((*rpl).c1.c1),
                               "m" ((*rpl).c1.c2),
                               "m" ((*rpl).c1.c3),
                               "m" ((*rpl).c2.c1),
                               "m" ((*rpl).c2.c2),
                               "m" ((*rpl).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

         rpl+=4;
         _prefetch_weyl_dble(rpl);
         rpl-=3;
         
         __asm__ __volatile__ ("addpd %%xmm0, %%xmm1 \n\t"
                               "addpd %%xmm2, %%xmm3 \n\t"
                               "addpd %%xmm4, %%xmm5 \n\t"
                               "addpd %%xmm1, %%xmm3 \n\t"
                               "addpd %%xmm3, %%xmm5 \n\t"
                               "movapd %%xmm5, %0"
                               :
                               "=m" (tmp1)
                               :
                               :
                               "xmm1", "xmm3", "xmm5");

         x+=(tmp1.c1+tmp1.c2);
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom==1)&&(NPROC>1))
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;
   }
   else
      return x;
}


double norm_square_dble(int vol,int icom,spinor_dble *pk)
{
   int n;
   double x,y;
   weyl_dble *rpk,*rpmax,*rptot;

   rpk=(weyl_dble*)(pk);
   rpmax=rpk;
   rptot=rpk+2*vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }

   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                               "movapd %1, %%xmm1 \n\t"
                               "movapd %2, %%xmm2 \n\t"
                               "movapd %3, %%xmm3 \n\t"
                               "movapd %4, %%xmm4 \n\t"
                               "movapd %5, %%xmm5"
                               :
                               :
                               "m" ((*rpk).c1.c1),
                               "m" ((*rpk).c1.c2),
                               "m" ((*rpk).c1.c3),
                               "m" ((*rpk).c2.c1),
                               "m" ((*rpk).c2.c2),
                               "m" ((*rpk).c2.c3)
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

         rpk+=4;
         _prefetch_weyl_dble(rpk);
         rpk-=3;
         
         __asm__ __volatile__ ("mulpd %%xmm0, %%xmm0 \n\t"
                               "mulpd %%xmm1, %%xmm1 \n\t"
                               "mulpd %%xmm2, %%xmm2 \n\t"
                               "mulpd %%xmm3, %%xmm3 \n\t"
                               "mulpd %%xmm4, %%xmm4 \n\t"
                               "mulpd %%xmm5, %%xmm5"
                               :
                               :
                               :
                               "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
         
         __asm__ __volatile__ ("addpd %%xmm0, %%xmm1 \n\t"
                               "addpd %%xmm2, %%xmm3 \n\t"
                               "addpd %%xmm4, %%xmm5 \n\t"
                               "addpd %%xmm1, %%xmm3 \n\t"
                               "addpd %%xmm3, %%xmm5 \n\t"
                               "movapd %%xmm5, %0"
                               :
                               "=m" (tmp1)
                               :
                               :
                               "xmm1", "xmm3", "xmm5");


         x+=(tmp1.c1+tmp1.c2);
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom==1)&&(NPROC>1))
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;
   }
   else
      return x;
}


void mulc_spinor_add_dble(int vol,spinor_dble *pk,spinor_dble *pl,
                          complex_dble z)
{
   weyl_dble *rpk,*rpl,*rptot;

   _sse_load_cmplx_dble(z);   
   rpk=(weyl_dble*)(pk);
   rpl=(weyl_dble*)(pl);
   rptot=rpk+2*vol;
   
   for (;rpk<rptot;)
   {
      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %%xmm0, %%xmm3 \n\t"
                            "movapd %%xmm1, %%xmm4 \n\t"
                            "movapd %%xmm2, %%xmm5"
                            :
                            :
                            "m" ((*rpl).c1.c1),
                            "m" ((*rpl).c1.c2),
                            "m" ((*rpl).c1.c3)
                            :
                            "xmm0", "xmm1", "xmm2");

      rpl+=4;
      _prefetch_weyl_dble(rpl);
      rpl-=4;
      
      __asm__ __volatile__ ("mulpd %%xmm6, %%xmm0 \n\t"
                            "mulpd %%xmm6, %%xmm1 \n\t"
                            "mulpd %%xmm6, %%xmm2 \n\t"
                            "shufpd $0x1, %%xmm3, %%xmm3 \n\t"
                            "shufpd $0x1, %%xmm4, %%xmm4 \n\t"
                            "shufpd $0x1, %%xmm5, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("addpd %0, %%xmm0 \n\t"
                            "addpd %1, %%xmm1 \n\t"
                            "addpd %2, %%xmm2 \n\t"
                            "mulpd %%xmm7, %%xmm3 \n\t"
                            "mulpd %%xmm7, %%xmm4 \n\t"
                            "mulpd %%xmm7, %%xmm5 \n\t"
                            "addpd %%xmm0, %%xmm3 \n\t"
                            "addpd %%xmm1, %%xmm4 \n\t"
                            "addpd %%xmm2, %%xmm5"                            
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2"
                            :
                            :
                            "m" ((*rpl).c2.c1),
                            "m" ((*rpl).c2.c2),
                            "m" ((*rpl).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2");

      
      __asm__ __volatile__ ("movapd %%xmm3, %0 \n\t"
                            "movapd %%xmm4, %1 \n\t"
                            "movapd %%xmm5, %2 \n\t"
                            "movapd %%xmm0, %%xmm3 \n\t"
                            "movapd %%xmm1, %%xmm4 \n\t"
                            "movapd %%xmm2, %%xmm5"
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3)
                            :
                            :
                            "xmm3", "xmm4", "xmm5");

      rpk+=4;
      _prefetch_weyl_dble(rpk);
      rpk-=4;      
      
      __asm__ __volatile__ ("mulpd %%xmm6, %%xmm0 \n\t"
                            "mulpd %%xmm6, %%xmm1 \n\t"
                            "mulpd %%xmm6, %%xmm2 \n\t"
                            "shufpd $0x1, %%xmm3, %%xmm3 \n\t"
                            "shufpd $0x1, %%xmm4, %%xmm4 \n\t"
                            "shufpd $0x1, %%xmm5, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("addpd %0, %%xmm0 \n\t"
                            "addpd %1, %%xmm1 \n\t"
                            "addpd %2, %%xmm2 \n\t"
                            "mulpd %%xmm7, %%xmm3 \n\t"
                            "mulpd %%xmm7, %%xmm4 \n\t"
                            "mulpd %%xmm7, %%xmm5 \n\t"
                            "addpd %%xmm0, %%xmm3 \n\t"
                            "addpd %%xmm1, %%xmm4 \n\t"
                            "addpd %%xmm2, %%xmm5"                            
                            :
                            :
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movapd %%xmm3, %0 \n\t"
                            "movapd %%xmm4, %1 \n\t"
                            "movapd %%xmm5, %2"
                            :
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=1;
      rpl+=1;
   }
}


void project_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   complex_dble z;

   z=spinor_prod_dble(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add_dble(vol,pk,pl,z);
}


double normalize_dble(int vol,int icom,spinor_dble *pk)
{
   double r,ri;
   weyl_dble *rpk,*rptot;

   r=norm_square_dble(vol,icom,pk);
   r=sqrt(r);

   if (error_loc(r==0.0,1,"normalize_dble [linalg_dble.c]",
                 "Vector has vanishing norm")!=0)
      return 0.0;

   ri=1.0/r;

   __asm__ __volatile__ ("movsd %0, %%xmm6 \n\t"
                         "unpcklpd %%xmm6, %%xmm6 \n\t"
                         "movapd %%xmm6, %%xmm7"
                         :
                         :
                         "m" (ri)
                         :
                         "xmm6", "xmm7");
   
   rpk=(weyl_dble*)(pk);
   rptot=rpk+2*vol;   
   
   for (;rpk<rptot;)
   {
      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %3, %%xmm3 \n\t"
                            "movapd %4, %%xmm4 \n\t"
                            "movapd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rpk+=4;
      _prefetch_weyl_dble(rpk);
      rpk-=4;      
      
      __asm__ __volatile__ ("mulpd %%xmm6, %%xmm0 \n\t"
                            "mulpd %%xmm7, %%xmm1 \n\t"
                            "mulpd %%xmm6, %%xmm2 \n\t"
                            "mulpd %%xmm7, %%xmm3 \n\t"
                            "mulpd %%xmm6, %%xmm4 \n\t"
                            "mulpd %%xmm7, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movapd %%xmm0, %0 \n\t"
                            "movapd %%xmm1, %1 \n\t"
                            "movapd %%xmm2, %2 \n\t"
                            "movapd %%xmm3, %3 \n\t"
                            "movapd %%xmm4, %4 \n\t"
                            "movapd %%xmm5, %5"                            
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3),                            
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=1;
   }

   return r;
}


void rotate_dble(int vol,int n,spinor_dble **ppk,complex_dble v[])
{
   int k,j,l,ix;
   complex_dble *z,*z0;
   su3_vector_dble *pk,*pj,*pj0;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)
            chid[k]=*(ppk[k]+ix);

         pk=(su3_vector_dble*)(psid);

         for (k=0;k<n;k++)  
         {
            pj0=(su3_vector_dble*)(chid);
            z0=&v[k];

            for (l=0;l<4;l++)
            {
               pj=pj0;
               z=z0;

               _sse_load_dble(*pj);
               _sse_load_cmplx_dble(*z);
               _sse_mulc_dble();
     
               for (j=1;j<n;j++)
               {
                  pj+=4;
                  z+=n;

                  _sse_load_dble(*pj);
                  _sse_load_cmplx_dble(*z);
                  _sse_add_mulc_re_dble();
                  _sse_load_dble(*pj);
                  _sse_add_mulc_im_dble();
               }

               _sse_store_up_dble(*pk);
               pk+=1;
               pj0+=1;
            }
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psid[k];
      }
   }
}


void mulg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;
   
   __asm__ __volatile__ ("movapd %0, %%xmm6 \n\t"
                         "movapd %%xmm6, %%xmm7"
                         :
                         :
                         "m" (sgn)
                         :
                         "xmm6", "xmm7");

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;)
   {
      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %3, %%xmm3 \n\t"
                            "movapd %4, %%xmm4 \n\t"
                            "movapd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
                            
      rpk+=8;
      _prefetch_weyl_dble(rpk);
      rpk-=8;      
      
      __asm__ __volatile__ ("mulpd %%xmm6, %%xmm0 \n\t"
                            "mulpd %%xmm7, %%xmm1 \n\t"
                            "mulpd %%xmm6, %%xmm2 \n\t"
                            "mulpd %%xmm7, %%xmm3 \n\t"
                            "mulpd %%xmm6, %%xmm4 \n\t"
                            "mulpd %%xmm7, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movapd %%xmm0, %0 \n\t"
                            "movapd %%xmm1, %1 \n\t"
                            "movapd %%xmm2, %2 \n\t"
                            "movapd %%xmm3, %3 \n\t"
                            "movapd %%xmm4, %4 \n\t"
                            "movapd %%xmm5, %5"                            
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3),                            
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=2;
   }
}


void mulmg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;
   
   __asm__ __volatile__ ("movapd %0, %%xmm6 \n\t"
                         "movapd %%xmm6, %%xmm7"
                         :
                         :
                         "m" (sgn)
                         :
                         "xmm6", "xmm7");

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;)
   {
      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %3, %%xmm3 \n\t"
                            "movapd %4, %%xmm4 \n\t"
                            "movapd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rpk+=8;
      _prefetch_weyl_dble(rpk);
      rpk-=8;
      
      __asm__ __volatile__ ("mulpd %%xmm6, %%xmm0 \n\t"
                            "mulpd %%xmm7, %%xmm1 \n\t"
                            "mulpd %%xmm6, %%xmm2 \n\t"
                            "mulpd %%xmm7, %%xmm3 \n\t"
                            "mulpd %%xmm6, %%xmm4 \n\t"
                            "mulpd %%xmm7, %%xmm5"
                            :
                            :
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      __asm__ __volatile__ ("movapd %%xmm0, %0 \n\t"
                            "movapd %%xmm1, %1 \n\t"
                            "movapd %%xmm2, %2 \n\t"
                            "movapd %%xmm3, %3 \n\t"
                            "movapd %%xmm4, %4 \n\t"
                            "movapd %%xmm5, %5"                            
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3),                            
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=2;
   }
}


void mulg5_spinor_sub_dble(int vol,spinor_dble *pk,spinor_dble *pl)
{
   weyl_dble *rpk,*rpl,*rpm;

   rpk=(weyl_dble*)(pk);
   rpl=(weyl_dble*)(pl);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;)
   {
      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %3, %%xmm3 \n\t"
                            "movapd %4, %%xmm4 \n\t"
                            "movapd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rpk+=8;
      _prefetch_spinor_dble(rpk);
      rpk-=8;
      
      __asm__ __volatile__ ("subpd %0, %%xmm0 \n\t"
                            "subpd %1, %%xmm1 \n\t"
                            "subpd %2, %%xmm2 \n\t"
                            "subpd %3, %%xmm3 \n\t"
                            "subpd %4, %%xmm4 \n\t"
                            "subpd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpl).c1.c1),
                            "m" ((*rpl).c1.c2),
                            "m" ((*rpl).c1.c3),
                            "m" ((*rpl).c2.c1),
                            "m" ((*rpl).c2.c2),
                            "m" ((*rpl).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("movapd %%xmm0, %0 \n\t"
                            "movapd %%xmm1, %1 \n\t"
                            "movapd %%xmm2, %2 \n\t"
                            "movapd %%xmm3, %3 \n\t"
                            "movapd %%xmm4, %4 \n\t"
                            "movapd %%xmm5, %5"                            
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3),                            
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=1;
      rpl+=1;

      __asm__ __volatile__ ("movapd %0, %%xmm0 \n\t"
                            "movapd %1, %%xmm1 \n\t"
                            "movapd %2, %%xmm2 \n\t"
                            "movapd %3, %%xmm3 \n\t"
                            "movapd %4, %%xmm4 \n\t"
                            "movapd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpk).c1.c1),
                            "m" ((*rpk).c1.c2),
                            "m" ((*rpk).c1.c3),
                            "m" ((*rpk).c2.c1),
                            "m" ((*rpk).c2.c2),
                            "m" ((*rpk).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      rpl+=8;
      _prefetch_spinor_dble(rpl);
      rpl-=8;
      
      __asm__ __volatile__ ("addpd %0, %%xmm0 \n\t"
                            "addpd %1, %%xmm1 \n\t"
                            "addpd %2, %%xmm2 \n\t"
                            "addpd %3, %%xmm3 \n\t"
                            "addpd %4, %%xmm4 \n\t"
                            "addpd %5, %%xmm5"
                            :
                            :
                            "m" ((*rpl).c1.c1),
                            "m" ((*rpl).c1.c2),
                            "m" ((*rpl).c1.c3),
                            "m" ((*rpl).c2.c1),
                            "m" ((*rpl).c2.c2),
                            "m" ((*rpl).c2.c3)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
      
      __asm__ __volatile__ ("movapd %%xmm0, %0 \n\t"
                            "movapd %%xmm1, %1 \n\t"
                            "movapd %%xmm2, %2 \n\t"
                            "movapd %%xmm3, %3 \n\t"
                            "movapd %%xmm4, %4 \n\t"
                            "movapd %%xmm5, %5"                            
                            :
                            "=m" ((*rpk).c1.c1),
                            "=m" ((*rpk).c1.c2),
                            "=m" ((*rpk).c1.c3),                            
                            "=m" ((*rpk).c2.c1),
                            "=m" ((*rpk).c2.c2),
                            "=m" ((*rpk).c2.c3));

      rpk+=1;
      rpl+=1;
   }
}

#elif (defined DH)
#include "dh.h"

complex_dble spinor_prod_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   complex_dble w,z;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   __alignx(16,rpk);
   __alignx(16,rpl);

   _dh_vector_declare2(1);
   _dh_vector_declare2(2);
   double _Complex sp;
   
   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
      smy[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      
      sp = __cmplx(0.0, 0.0);

      for (;rpk<rpmax;)
      {
         _dh_vector_load2((*rpk).c1,(*rpl).c1,1);
         _dh_vector_load2((*rpk).c2,(*rpl).c2,2);

         _dh_vector_prod2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);


         _dh_vector_load2((*rpk).c3,(*rpl).c3,1);
         _dh_vector_load2((*rpk).c4,(*rpl).c4,2);

         _dh_vector_prod2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=__creal(sp);
      smy[0]+=__cimag(sp);

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];
         smy[n]+=smy[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
         smy[n-1]=0.0;
      }
   }

   x=0.0;
   y=0.0;

   for (n=0;n<MAX_LEVELS;n++)
   {
      x+=smx[n];
      y+=smy[n];
   }

   if ((icom!=1)||(NPROC==1))
   {
      z.re=x;
      z.im=y;
   }
   else
   {
      w.re=x;
      w.im=y;      
      MPI_Reduce(&w.re,&z.re,2,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&z.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);  
   }

   return z; 
}


double spinor_prod_re_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   __alignx(16,rpk);
   __alignx(16,rpl);

   _dh_vector_declare2(1);
   _dh_vector_declare2(2);
   double _Complex sp;
   
   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      
      sp = __cmplx(0.0, 0.0);

      for (;rpk<rpmax;)
      {
         _dh_vector_load2((*rpk).c1,(*rpl).c1,1);
         _dh_vector_load2((*rpk).c2,(*rpl).c2,2);

         _dh_vector_prod2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);


         _dh_vector_load2((*rpk).c3,(*rpl).c3,1);
         _dh_vector_load2((*rpk).c4,(*rpl).c4,2);

         _dh_vector_prod2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=__creal(sp);

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;   
   }
}

 
double norm_square_dble(int vol,int icom,spinor_dble *pk)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpmax,*rptot;

   __alignx(16,rpk);

   _dh_vector_declare(1);
   _dh_vector_declare(2);
   double _Complex sp;

   rpk=pk;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      
      sp = __cmplx(0.0, 0.0);

      for (;rpk<rpmax;)
      {
         _dh_vector_load((*rpk).c1,1);
         _dh_vector_load((*rpk).c2,2);

         _dh_vector_square2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);


         _dh_vector_load((*rpk).c3,1);
         _dh_vector_load((*rpk).c4,2);

         _dh_vector_square2(1,2);
         _dh_vector_addu(sp,1);
         _dh_vector_addu(sp,2);

         rpk+=1;
      }

      cnt[0]+=1;
      smx[0]+=__creal(sp);

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;    
   }
}


void mulc_spinor_add_dble(int vol,spinor_dble *pk,spinor_dble *pl,
                          complex_dble z)
{
   spinor_dble *rpk,*rpl;

   __alignx(16,rpk);
   __alignx(16,rpl);

   _dh_vector_declare2(1);
   _dh_vector_declare2(2);
   double _Complex dcz;
   
   rpk=pk;
   rpl=pl;
   
   dcz = __lfpd(&z.re);
   for (;rpk<(pk+vol);)
   {
      _dh_vector_load2((*rpk).c1,(*rpl).c1,1);
      _dh_vector_load2((*rpk).c2,(*rpl).c2,2);

      _dh_vector_mulc_assign2(dcz,1,2);

      _dh_vector_storeu((*rpk).c1,1);
      _dh_vector_storeu((*rpk).c2,2);


      _dh_vector_load2((*rpk).c3,(*rpl).c3,1);
      _dh_vector_load2((*rpk).c4,(*rpl).c4,2);

      _dh_vector_mulc_assign2(dcz,1,2);

      _dh_vector_storeu((*rpk).c3,1);
      _dh_vector_storeu((*rpk).c4,2);

      rpk+=1;
      rpl+=1;
   }
}


void project_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   complex_dble z;

   z=spinor_prod_dble(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add_dble(vol,pk,pl,z);
}


double normalize_dble(int vol,int icom,spinor_dble *pk)
{
   double r;
   spinor_dble *rpk;
   complex_dble z;

   __alignx(16,rpk);

   _dh_vector_declare2(1);
   double _Complex dcz;

   r=norm_square_dble(vol,icom,pk);
   r=sqrt(r);
   
   if (error_loc(r==0.0,1,"normalize_dble [linalg_dble.c]",
       "Vector has vanishing norm")!=0)
      return 0.0;

   z.re=1.0/r;
   rpk=pk;

   dcz = __lfpd(&z.re);
   for (;rpk<(pk+vol);)
   {
      _dh_vector_load2((*rpk).c1,(*rpk).c2,1);
      _dh_vector_mul(dcz,1);
      _dh_vector_storeu((*rpk).c1,1);
      _dh_vector_storet((*rpk).c2,1);

      _dh_vector_load2((*rpk).c3,(*rpk).c4,1);
      _dh_vector_mul(dcz,1);
      _dh_vector_storeu((*rpk).c3,1);
      _dh_vector_storet((*rpk).c4,1);

      rpk+=1;
   }

   return r;
}


void rotate_dble(int vol,int n,spinor_dble **ppk,complex_dble v[])
{
   int k,j,ix;
   complex_dble *z;
   spinor_dble *pk,*pj;

   __alignx(16,pk);
   __alignx(16,pj);

   _dh_vector_declare(1);
   _dh_vector_declare(2);
   _dh_vector_declare(3);
   _dh_vector_declare(4);
   double _Complex dcz;
   
   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psid[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            dcz = __lfpd(&(*z).re);
            _dh_vector_load((*pj).c1,1);
            _dh_vector_load((*pj).c2,2);
            _dh_vector_load((*pj).c3,3);
            _dh_vector_load((*pj).c4,4);

            _dh_vector_mulc2(dcz,1,2);
            _dh_vector_mulc2(dcz,3,4);
            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;

               dcz = __lfpd(&(*z).re);
               _dh_vector_load((*pj).c1,1);
               _dh_vector_load((*pj).c2,2);
               _dh_vector_load((*pj).c3,3);
               _dh_vector_load((*pj).c4,4);

               _dh_vector_mulc_add2(dcz,1,2);
               _dh_vector_mulc_add2(dcz,3,4);
            }
            _dh_vector_storeu((*pk).c1,1);
            _dh_vector_storeu((*pk).c2,2);
            _dh_vector_storeu((*pk).c3,3);
            _dh_vector_storeu((*pk).c4,4);
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psid[k];
      }
   }
}


void mulg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   __alignx(16,rpk);

   _dh_vector_declareu(1);
   _dh_vector_declareu(2);

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;

   for (;rpk<rpm;rpk+=2)
   {
      _dh_vector_load((*rpk).c1,1);
      _dh_vector_load((*rpk).c2,2);

      _dh_vector_neg(1);
      _dh_vector_neg(2);

      _dh_vector_storeu((*rpk).c1,1);
      _dh_vector_storeu((*rpk).c2,2);
   }
}


void mulmg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   __alignx(16,rpk);

   _dh_vector_declareu(1);
   _dh_vector_declareu(2);

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;

   for (;rpk<rpm;rpk+=2)
   {
      _dh_vector_load((*rpk).c1,1);
      _dh_vector_load((*rpk).c2,2);

      _dh_vector_neg(1);
      _dh_vector_neg(2);

      _dh_vector_storeu((*rpk).c1,1);
      _dh_vector_storeu((*rpk).c2,2);
   }
}


void mulg5_spinor_sub_dble(int vol,spinor_dble *pk,spinor_dble *pl)
{
   spinor_dble *pkm;

   __alignx(16,pk);
   
   _dh_vector_declare1(1);
   _dh_vector_declare1(2);
   _dh_vector_declare1(3);
   _dh_vector_declare1(4);
   
   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      _dh_vector_load((*pk).c1,1);
      _dh_vector_load((*pk).c2,2);
      _dh_vector_load((*pl).c1,3);
      _dh_vector_load((*pl).c2,4);
      
      _dh_vector_sub_assign(1,3);
      _dh_vector_sub_assign(2,4);
      
      _dh_vector_store((*pk).c1,1);
      _dh_vector_store((*pk).c2,2);
      
      _dh_vector_load((*pk).c3,1);
      _dh_vector_load((*pk).c4,2);
      _dh_vector_load((*pl).c3,3);
      _dh_vector_load((*pl).c4,4);
      
      _dh_vector_add_assign(1,3);
      _dh_vector_add_assign(2,4);
     
      _dh_vector_store((*pk).c3,1);
      _dh_vector_store((*pk).c4,2);

      pl+=1;
   }
}

#elif (defined QPX)
#include "qpx.h"
complex_dble spinor_prod_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   complex_dble w,z;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
      smy[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;
      y=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpl).c1)+
               _vector_prod_re((*rpk).c2,(*rpl).c2)+
               _vector_prod_re((*rpk).c3,(*rpl).c3)+
               _vector_prod_re((*rpk).c4,(*rpl).c4));

         y+=(_vector_prod_im((*rpk).c1,(*rpl).c1)+
               _vector_prod_im((*rpk).c2,(*rpl).c2)+
               _vector_prod_im((*rpk).c3,(*rpl).c3)+
               _vector_prod_im((*rpk).c4,(*rpl).c4));

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;
      smy[0]+=y;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];
         smy[n]+=smy[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
         smy[n-1]=0.0;
      }
   }

   x=0.0;
   y=0.0;

   for (n=0;n<MAX_LEVELS;n++)
   {
      x+=smx[n];
      y+=smy[n];
   }

   if ((icom!=1)||(NPROC==1))
   {
      z.re=x;
      z.im=y;
   }
   else
   {
      w.re=x;
      w.im=y;      
      MPI_Allreduce(&w.re,&z.re,2,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD);
      //MPI_Bcast(&z.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);  
   }

   return z; 
}


double spinor_prod_re_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpl).c1)+
               _vector_prod_re((*rpk).c2,(*rpl).c2)+
               _vector_prod_re((*rpk).c3,(*rpl).c3)+
               _vector_prod_re((*rpk).c4,(*rpl).c4));

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD);
      //MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;   
   }
}

 
double norm_square_dble(int vol,int icom,spinor_dble *pk)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpmax,*rptot;

   rpk=pk;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpk).c1)+
               _vector_prod_re((*rpk).c2,(*rpk).c2)+
               _vector_prod_re((*rpk).c3,(*rpk).c3)+
               _vector_prod_re((*rpk).c4,(*rpk).c4));

         rpk+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD);
      //MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;    
   }
}

static complex_dble cc __attribute__((aligned(32)));

//qpx
void mulc_spinor_add_dble(int vol,spinor_dble * __restrict__ pk,spinor_dble * __restrict__ pl,
                          complex_dble z)
{
  
  _qpx_vector_declare(zv);
  _qpx_raw_spinor_declare(vio);
  _qpx_raw_spinor_declare(vin);

  _qpx_raw_spinor_declare(vio2);
  _qpx_raw_spinor_declare(vin2);
  
  int u2do, rest;
  int i;
  int j;

  cc = z;
  _qpx_dcomplex_load(zv, cc);
  
  u2do = vol/2;
  rest = vol%2;
  //rest = vol;
 
  //unroll 2
  for(i=0;i<u2do;i++) {
    j=2*i;
    _qpx_raw_spinor_load(vio, pk[j]);
    _qpx_raw_spinor_load(vin, pl[j]);
    _qpx_raw_spinor_load(vio2, pk[j+1]);
    _qpx_raw_spinor_load(vin2, pl[j+1]);
    
    _qpx_raw_spinor_mcsa(vio, vin, zv);
    _qpx_raw_spinor_mcsa(vio2, vin2, zv);
    
    _qpx_raw_spinor_store(vio, pk[j]);
    _qpx_raw_spinor_store(vio2, pk[j+1]);
    
  }

  //j=0;
  
  if(rest) {
  
    for(i=j;i<vol;i++) {
      _qpx_raw_spinor_load(vio, pk[i]);
      _qpx_raw_spinor_load(vin, pl[i]);
      _qpx_raw_spinor_mcsa(vio, vin, zv);
      _qpx_raw_spinor_store(vio, pk[i]);
      
    }
  }

}


void project_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   complex_dble z;

   z=spinor_prod_dble(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add_dble(vol,pk,pl,z);
}


double normalize_dble(int vol,int icom,spinor_dble *pk)
{
   double r,ri;
   spinor_dble *rpk;

   r=norm_square_dble(vol,icom,pk);
   r=sqrt(r);
   
   if (error_loc(r==0.0,1,"normalize_dble [linalg_dble.c]",
       "Vector has vanishing norm")!=0)
      return 0.0;

   ri=1.0/r;
   rpk=pk;
   
   for (;rpk<(pk+vol);)
   {
      _vector_mul((*rpk).c1,ri,(*rpk).c1);
      _vector_mul((*rpk).c2,ri,(*rpk).c2);
      _vector_mul((*rpk).c3,ri,(*rpk).c3);
      _vector_mul((*rpk).c4,ri,(*rpk).c4);

      rpk+=1;
   }

   return r;
}


void rotate_dble(int vol,int n,spinor_dble **ppk,complex_dble v[])
{
   int k,j,ix;
   complex_dble *z;
   spinor_dble *pk,*pj;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psid[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            _vector_mulc((*pk).c1,*z,(*pj).c1);
            _vector_mulc((*pk).c2,*z,(*pj).c2);
            _vector_mulc((*pk).c3,*z,(*pj).c3);
            _vector_mulc((*pk).c4,*z,(*pj).c4);
     
            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;

               _vector_mulc_assign((*pk).c1,*z,(*pj).c1);
               _vector_mulc_assign((*pk).c2,*z,(*pj).c2);
               _vector_mulc_assign((*pk).c3,*z,(*pj).c3);
               _vector_mulc_assign((*pk).c4,*z,(*pj).c4);            
            }
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psid[k];
      }
   }
}


void mulg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulmg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulg5_spinor_sub_dble(int vol,spinor_dble *pk,spinor_dble *pl)
{
   spinor_dble *pkm;

   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      _vector_sub_assign((*pk).c1,(*pl).c1);
      _vector_sub_assign((*pk).c2,(*pl).c2);
      _vector_add_assign((*pk).c3,(*pl).c3);
      _vector_add_assign((*pk).c4,(*pl).c4);      

      pl+=1;
   }
}

#else

complex_dble spinor_prod_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   complex_dble w,z;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
      smy[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;
      y=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpl).c1)+
               _vector_prod_re((*rpk).c2,(*rpl).c2)+
               _vector_prod_re((*rpk).c3,(*rpl).c3)+
               _vector_prod_re((*rpk).c4,(*rpl).c4));

         y+=(_vector_prod_im((*rpk).c1,(*rpl).c1)+
               _vector_prod_im((*rpk).c2,(*rpl).c2)+
               _vector_prod_im((*rpk).c3,(*rpl).c3)+
               _vector_prod_im((*rpk).c4,(*rpl).c4));

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;
      smy[0]+=y;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];
         smy[n]+=smy[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
         smy[n-1]=0.0;
      }
   }

   x=0.0;
   y=0.0;

   for (n=0;n<MAX_LEVELS;n++)
   {
      x+=smx[n];
      y+=smy[n];
   }

   if ((icom!=1)||(NPROC==1))
   {
      z.re=x;
      z.im=y;
   }
   else
   {
      w.re=x;
      w.im=y;      
      MPI_Allreduce(&w.re,&z.re,2,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD);
      //MPI_Bcast(&z.re,2,MPI_DOUBLE,0,MPI_COMM_WORLD);  
   }

   return z; 
}


double spinor_prod_re_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpl,*rpmax,*rptot;

   rpk=pk;
   rpl=pl;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpl).c1)+
               _vector_prod_re((*rpk).c2,(*rpl).c2)+
               _vector_prod_re((*rpk).c3,(*rpl).c3)+
               _vector_prod_re((*rpk).c4,(*rpl).c4));

         rpk+=1;
         rpl+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;   
   }
}

 
double norm_square_dble(int vol,int icom,spinor_dble *pk)
{
   int n;
   double x,y;
   spinor_dble *rpk,*rpmax,*rptot;

   rpk=pk;
   rpmax=rpk;
   rptot=rpk+vol;

   for (n=0;n<MAX_LEVELS;n++)
   {
      cnt[n]=0;
      smx[n]=0.0;
   }
   
   for (;rpmax<rptot;)
   {
      rpmax+=BLK_LENGTH;
      if (rpmax>rptot)
         rpmax=rptot;
      x=0.0;

      for (;rpk<rpmax;)
      {
         x+=(_vector_prod_re((*rpk).c1,(*rpk).c1)+
               _vector_prod_re((*rpk).c2,(*rpk).c2)+
               _vector_prod_re((*rpk).c3,(*rpk).c3)+
               _vector_prod_re((*rpk).c4,(*rpk).c4));

         rpk+=1;
      }

      cnt[0]+=1;
      smx[0]+=x;

      for (n=1;(cnt[n-1]>=BLK_LENGTH)&&(n<MAX_LEVELS);n++)
      {
         cnt[n]+=1;
         smx[n]+=smx[n-1];

         cnt[n-1]=0;
         smx[n-1]=0.0;
      }
   }

   x=0.0;

   for (n=0;n<MAX_LEVELS;n++)
      x+=smx[n];

   if ((icom!=1)||(NPROC==1))
      return x;
   else
   {
      MPI_Reduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
      MPI_Bcast(&y,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
      return y;    
   }
}


void mulc_spinor_add_dble(int vol,spinor_dble *pk,spinor_dble *pl,
                          complex_dble z)
{
   spinor_dble *rpk,*rpl;

   rpk=pk;
   rpl=pl;
   
   for (;rpk<(pk+vol);)
   {
      _vector_mulc_assign((*rpk).c1,z,(*rpl).c1);
      _vector_mulc_assign((*rpk).c2,z,(*rpl).c2);
      _vector_mulc_assign((*rpk).c3,z,(*rpl).c3);
      _vector_mulc_assign((*rpk).c4,z,(*rpl).c4);

      rpk+=1;
      rpl+=1;
   }
}


void project_dble(int vol,int icom,spinor_dble *pk,spinor_dble *pl)
{
   complex_dble z;

   z=spinor_prod_dble(vol,icom,pl,pk);
   z.re=-z.re;
   z.im=-z.im;
   mulc_spinor_add_dble(vol,pk,pl,z);
}


double normalize_dble(int vol,int icom,spinor_dble *pk)
{
   double r,ri;
   spinor_dble *rpk;

   r=norm_square_dble(vol,icom,pk);
   r=sqrt(r);
   
   if (error_loc(r==0.0,1,"normalize_dble [linalg_dble.c]",
       "Vector has vanishing norm")!=0)
      return 0.0;

   ri=1.0/r;
   rpk=pk;
   
   for (;rpk<(pk+vol);)
   {
      _vector_mul((*rpk).c1,ri,(*rpk).c1);
      _vector_mul((*rpk).c2,ri,(*rpk).c2);
      _vector_mul((*rpk).c3,ri,(*rpk).c3);
      _vector_mul((*rpk).c4,ri,(*rpk).c4);

      rpk+=1;
   }

   return r;
}


void rotate_dble(int vol,int n,spinor_dble **ppk,complex_dble v[])
{
   int k,j,ix;
   complex_dble *z;
   spinor_dble *pk,*pj;

   if ((n>nrot)&&(ifail==0))
      alloc_wrotate(n);

   if ((n>0)&&(ifail==0))
   {
      for (ix=0;ix<vol;ix++)
      {
         for (k=0;k<n;k++)  
         {
            pk=&(psid[k]);
            pj=ppk[0]+ix;
            z=&v[k];

            _vector_mulc((*pk).c1,*z,(*pj).c1);
            _vector_mulc((*pk).c2,*z,(*pj).c2);
            _vector_mulc((*pk).c3,*z,(*pj).c3);
            _vector_mulc((*pk).c4,*z,(*pj).c4);
     
            for (j=1;j<n;j++)
            {
               pj=ppk[j]+ix;
               z+=n;

               _vector_mulc_assign((*pk).c1,*z,(*pj).c1);
               _vector_mulc_assign((*pk).c2,*z,(*pj).c2);
               _vector_mulc_assign((*pk).c3,*z,(*pj).c3);
               _vector_mulc_assign((*pk).c4,*z,(*pj).c4);            
            }
         }

         for (k=0;k<n;k++)
            *(ppk[k]+ix)=psid[k];
      }
   }
}


void mulg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   rpk+=1;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulmg5_dble(int vol,spinor_dble *pk)
{
   weyl_dble *rpk,*rpm;

   rpk=(weyl_dble*)(pk);
   rpm=rpk+2*vol;
   
   for (;rpk<rpm;rpk+=2)
   {
      (*rpk).c1.c1.re=-(*rpk).c1.c1.re;
      (*rpk).c1.c1.im=-(*rpk).c1.c1.im;      
      (*rpk).c1.c2.re=-(*rpk).c1.c2.re;
      (*rpk).c1.c2.im=-(*rpk).c1.c2.im;      
      (*rpk).c1.c3.re=-(*rpk).c1.c3.re;
      (*rpk).c1.c3.im=-(*rpk).c1.c3.im;      
      (*rpk).c2.c1.re=-(*rpk).c2.c1.re;
      (*rpk).c2.c1.im=-(*rpk).c2.c1.im;      
      (*rpk).c2.c2.re=-(*rpk).c2.c2.re;
      (*rpk).c2.c2.im=-(*rpk).c2.c2.im;      
      (*rpk).c2.c3.re=-(*rpk).c2.c3.re;
      (*rpk).c2.c3.im=-(*rpk).c2.c3.im;      
   }
}


void mulg5_spinor_sub_dble(int vol,spinor_dble *pk,spinor_dble *pl)
{
   spinor_dble *pkm;

   pkm=pk+vol;
   
   for (;pk<pkm;pk++)
   {
      _vector_sub_assign((*pk).c1,(*pl).c1);
      _vector_sub_assign((*pk).c2,(*pl).c2);
      _vector_add_assign((*pk).c3,(*pl).c3);
      _vector_add_assign((*pk).c4,(*pl).c4);      

      pl+=1;
   }
}

#endif

