
/*******************************************************************************
*
* File Qhat.c
*
* Copyright (C) 2005 Martin Luescher, 2007 Bjoern Leder, 2010 Andrea Nobile
*
* This software is distributed under the terms of the GNU General Public
* License (GPL)
*
* Application of the even-odd preconditioned Wilson-Dirac operator
*
* The externally accessible functions are
*
*   void Qhat(int k,int l)
*     Applies Qhat to the global single-precision field *ps[k][] and assigns
*     the result to *ps[l][]
*
*   void Qoe(int k,int l)
*     Applies the operator Qoe to the even part of the global single-precision
*     field *ps[k][] and assigns the result to *ps[l][]
*
*   void Qeo(int k,int l)
*     Applies the operator Qeo to the odd part of the global single-precision
*     field *ps[k][] and *subtracts* the result from *ps[l][]
*
*   void Qnohat(int k,int l)
*     Applies the full Dirac operator Q to the global single-precision field
*     *ps[k][] and assigns the result to *ps[l][]
*
*   void Qhat_blk(block_t *b,int k,int l)
*     Applies Qhat to the single-precision field b.s[k][] on the block b and
*     assigns the result to b.s[l][]
*
*   void Qoe_blk(block_t *b,int k,int l)
*     Applies the operator Qoe to the even part of the single-precision field
*     b.s[k][] on the block b and assigns the result to b.s[l][]
*
*   void Qeo_blk(block_t *b,int k,int l)
*     Applies the operator Qeo to the odd part of the single-precision field
*     b.s[k][] on the block b and *subtracts* the result from b.s[l][]
*
*   void Qnohat_blk(block_t *b,int k,int l)
*     Applies the full Dirac operator Q to the single-precision field b.s[k][]
*     on the block b and assigns the result to b.s[l][]
*
* Notes:
*
* The notation and normalization conventions are specified in the notes
* "Implementation of the lattice Dirac operator" (file doc/dirac.ps)
*
* In all these programs it is assumed that Qee and Qoo (or 1/Qoo in the
* case of the preconditioned operator) are stored in the appropriate sw
* arrays. Whether the field arrays are in the proper condition can normally
* be determined by querying the flags data base [see flags.c]
*
* All non-block programs in this file act globally and should be called
* from all processes with the same parameters. Communication buffers are
* allocated automatically when needed
*
* The block programs, on the other hand, act locally. Any errors occuring in
* these are recorded by the error_loc [utils.c] function
*
*******************************************************************************/

#define QHAT_C

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "mpi.h"
#include "su3.h"
#include "flags.h"
#include "start.h"
#include "block.h"
#include "sw_term.h"
#include "dirac.h"
#include "global.h"

static float coe,ceo;
static const spinor s0={{{0.0f}}};

#if (defined SSE)
#include "sse.h"

static spinor rs __attribute__ ((aligned (16)));
//static sse_vector r12,r34;


#ifdef __SSE__
#include <xmmintrin.h>
#endif

#ifdef __SSE2__
#include <emmintrin.h>
#endif

#ifdef __SSE3__
#include <pmmintrin.h>
#endif


#define _load_cst(c) \
__asm__ __volatile__ ("movss %0, %%xmm6 \n\t" \
                      "shufps $0x0, %%xmm6, %%xmm6" \
                      : \
                      : \
                      "m" (c) \
                      : \
                      "xmm6")

#define _load_csts(c) \
__asm__ __volatile__ ("movss %0, %%xmm6 \n\t" \
                      "shufps $0x0, %%xmm6, %%xmm6 \n\t" \
                      "movaps %%xmm6, %%xmm7 \n\t" \
                      "mulps %1, %%xmm7" \
                      : \
                      : \
                      "m" (c), \
                      "m" (_sse_sgn) \
                      : \
                      "xmm6", "xmm7")

#define _mul_cst() \
__asm__ __volatile__ ("mulps %%xmm6, %%xmm0 \n\t" \
                      "mulps %%xmm6, %%xmm1 \n\t" \
                      "mulps %%xmm6, %%xmm2" \
                      : \
                      : \
                      : \
                      "xmm0", "xmm1", "xmm2")

#define _mul_cst_up() \
__asm__ __volatile__ ("mulps %%xmm6, %%xmm3 \n\t" \
                      "mulps %%xmm6, %%xmm4 \n\t" \
                      "mulps %%xmm6, %%xmm5" \
                      : \
                      : \
                      : \
                      "xmm3", "xmm4", "xmm5")

#define _mul_csts() \
__asm__ __volatile__ ("mulps %%xmm7, %%xmm0 \n\t" \
                      "mulps %%xmm7, %%xmm1 \n\t" \
                      "mulps %%xmm7, %%xmm2" \
                      : \
                      : \
                      : \
                      "xmm0", "xmm1", "xmm2")


static void qoe(int *piup,int *pidn,su3 *u,spinor *pk)
{
   spinor *sp,*sm;
   sse_vector r12,r34;
/******************************* direction +0 *********************************/

   sp=pk+(*(piup++));

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_pair_load_up((*sp).c3,(*sp).c4);

   sm=pk+(*(pidn++));
   _prefetch_spinor(sm);

   _sse_vector_add();
   sp=pk+(*(piup++));
   _prefetch_spinor(sp);
   _sse_su3_multiply(*u);

   _sse_vector_store_up(r12);
   _sse_vector_store_up(r34);

/******************************* direction -0 *********************************/

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_pair_load_up((*sm).c3,(*sm).c4);

   u+=2;
   _prefetch_su3_dble(u);
   u-=1;
   _sse_vector_sub();
   sm=pk+(*(pidn++));
   _prefetch_spinor(sm);
   _sse_su3_inverse_multiply(*u);

   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_sub();
   _sse_vector_store(r34);

/******************************* direction +1 *********************************/

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_pair_load_up((*sp).c4,(*sp).c3);

   _sse_vector_i_add();
   sp=pk+(*(piup++));
   _prefetch_spinor(sp);
   u+=1;
   _sse_su3_multiply(*u);

   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_xch_i_sub();
   _sse_vector_store(r34);

/******************************* direction -1 *********************************/

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_pair_load_up((*sm).c4,(*sm).c3);

   u+=2;
   _prefetch_su3_dble(u);
   u-=1;
   _sse_vector_i_sub();
   sm=pk+(*(pidn++));
   _prefetch_spinor(sm);
   _sse_su3_inverse_multiply(*u);

   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_xch_i_add();
   _sse_vector_store(r34);

/******************************* direction +2 *********************************/

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_pair_load_up((*sp).c4,(*sp).c3);

   _sse_vector_addsub();

   u+=1;
   _sse_su3_multiply(*u);
   sp=pk+(*(piup));
   _prefetch_spinor(sp);
   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_xch();
   _sse_vector_subadd();
   _sse_vector_store(r34);

/******************************* direction -2 *********************************/

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_pair_load_up((*sm).c4,(*sm).c3);

   u+=2;
   _prefetch_su3_dble(u);
   u-=1;
   _sse_vector_subadd();
   sm=pk+(*(pidn));
   _prefetch_spinor(sm);
   _sse_su3_inverse_multiply(*u);

   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_xch();
   _sse_vector_addsub();
   _sse_vector_store(r34);

/******************************* direction +3 *********************************/

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_pair_load_up((*sp).c3,(*sp).c4);

   _sse_vector_i_addsub();
   u+=1;
   _sse_su3_multiply(*u);

   _sse_vector_load(r12);
   _sse_vector_add();
   _sse_vector_store(r12);

   _sse_vector_load(r34);
   _sse_vector_i_subadd();
   _sse_vector_store(r34);

/******************************* direction -3 *********************************/

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_pair_load_up((*sm).c3,(*sm).c4);

   u+=2;
   _prefetch_su3_dble(u);
   u-=1;
   _sse_vector_i_subadd();
   _sse_su3_inverse_multiply(*u);

   _load_csts(coe);
   _sse_vector_load(r12);
   _sse_vector_add();
   _mul_csts();
   _sse_pair_store(rs.c1,rs.c2);

   _sse_vector_load(r34);
   _sse_vector_i_addsub();
   _mul_cst();
   _sse_pair_store(rs.c3,rs.c4);
}


static void qeo(int *piup,int *pidn,su3 *u,spinor *pl)
{
   spinor *sp,*sm;
   sse_vector r12,r34;
/******************************* direction +0 *********************************/

   sp=pl+(*(piup++));
   _prefetch_spinor(sp);

   _sse_pair_load(rs.c1,rs.c2);
   _sse_pair_load_up(rs.c3,rs.c4);
   _sse_vector_store(r12);
   _sse_vector_store_up(r34);

   sm=pl+(*(pidn++));
   _prefetch_spinor(sm);
   _sse_vector_sub();
   _sse_su3_inverse_multiply(*u);

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_vector_sub();
   _sse_pair_store((*sp).c1,(*sp).c2);

   _sse_pair_load((*sp).c3,(*sp).c4);
   _sse_vector_sub();
   _sse_pair_store((*sp).c3,(*sp).c4);

/******************************* direction -0 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sp=pl+(*(piup++));
   _prefetch_spinor(sp);
   _sse_vector_add();
   u+=1;
   _sse_su3_multiply(*u);

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_vector_sub();
   _sse_pair_store((*sm).c1,(*sm).c2);

   _sse_pair_load((*sm).c3,(*sm).c4);
   _sse_vector_add();
   _sse_pair_store((*sm).c3,(*sm).c4);

/******************************* direction +1 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sm=pl+(*(pidn++));
   _prefetch_spinor(sm);
   _sse_vector_xch_i_sub();
   u+=1;
   _sse_su3_inverse_multiply(*u);

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_vector_sub();
   _sse_pair_store((*sp).c1,(*sp).c2);

   _sse_pair_load((*sp).c3,(*sp).c4);
   _sse_vector_xch_i_add();
   _sse_pair_store((*sp).c3,(*sp).c4);

/******************************* direction -1 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sp=pl+(*(piup++));
   _prefetch_spinor(sp);
   _sse_vector_xch_i_add();
   u+=1;
   _sse_su3_multiply(*u);

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_vector_sub();
   _sse_pair_store((*sm).c1,(*sm).c2);

   _sse_pair_load((*sm).c3,(*sm).c4);
   _sse_vector_xch_i_sub();
   _sse_pair_store((*sm).c3,(*sm).c4);

/******************************* direction +2 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sm=pl+(*(pidn++));
   _prefetch_spinor(sm);
   _sse_vector_xch();
   _sse_vector_subadd();
   u+=1;
   _sse_su3_inverse_multiply(*u);

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_vector_sub();
   _sse_pair_store((*sp).c1,(*sp).c2);

   _sse_pair_load((*sp).c3,(*sp).c4);
   _sse_vector_xch();
   _sse_vector_addsub();
   _sse_pair_store((*sp).c3,(*sp).c4);

/******************************* direction -2 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sp=pl+(*(piup));
   _prefetch_spinor(sp);
   _sse_vector_xch();
   _sse_vector_addsub();
   u+=1;   
   _sse_su3_multiply(*u);

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_vector_sub();
   _sse_pair_store((*sm).c1,(*sm).c2);

   _sse_pair_load((*sm).c3,(*sm).c4);
   _sse_vector_xch();
   _sse_vector_subadd();
   _sse_pair_store((*sm).c3,(*sm).c4);

/******************************* direction +3 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   sm=pl+(*(pidn));
   _prefetch_spinor(sm);
   _sse_vector_i_subadd();
   u+=1;   
   _sse_su3_inverse_multiply(*u);

   _sse_pair_load((*sp).c1,(*sp).c2);
   _sse_vector_sub();
   _sse_pair_store((*sp).c1,(*sp).c2);

   _sse_pair_load((*sp).c3,(*sp).c4);
   _sse_vector_i_addsub();
   _sse_pair_store((*sp).c3,(*sp).c4);

/******************************* direction -3 *********************************/

   _sse_vector_load(r12);
   _sse_vector_load_up(r34);

   _sse_vector_i_addsub();
   u+=1;
   _sse_su3_multiply(*u);

   _sse_pair_load((*sm).c1,(*sm).c2);
   _sse_vector_sub();
   _sse_pair_store((*sm).c1,(*sm).c2);

   _sse_pair_load((*sm).c3,(*sm).c4);
   _sse_vector_i_subadd();
   _sse_pair_store((*sm).c3,(*sm).c4);
}

#elif (defined DH)
#include "dh.h"

#define _dh_static_vector_declare(a) \
              static double _Complex  dcr##a##1,dcr##a##2,dcr##a##3

_dh_static_vector_declare(rs1);
_dh_static_vector_declare(rs2);
_dh_static_vector_declare(rs3);
_dh_static_vector_declare(rs4);

static void qoe(int *piup,int *pidn,su3 *u,spinor *pk)
{
   spinor *sp,*sm;

#pragma disjoint(*u,*sp,*sm)
   __alignx(16,sp);
   __alignx(16,sm);
   __alignx(16,u);

   _dh_vector_declare(1);
   _dh_vector_declare(2);
   _dh_su3_declare(1);

/******************************* direction +0 *********************************/

      sp=pk+(*(piup++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sp).c1,1);
      _dhs_vector_load((*sp).c3,2);

      _dh_vector_add(1,1,2);

      _dhs_vector_load((*sp).c2,1);
      _dhs_vector_load((*sp).c4,2);

      _dh_vector_add(2,1,2);

      _dh_su3_multiply(1,2,1);

      u+=1;

      _dh_vector_assign(rs1,1);
      _dh_vector_assign(rs2,2);
      _dh_vector_assign(rs3,1);
      _dh_vector_assign(rs4,2);

/******************************* direction -0 *********************************/

      sm=pk+(*(pidn++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sm).c1,1);
      _dhs_vector_load((*sm).c3,2);

      _dh_vector_sub(1,1,2);

      _dhs_vector_load((*sm).c2,1);
      _dhs_vector_load((*sm).c4,2);

      _dh_vector_sub(2,1,2);

      _dh_su3_inverse_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_sub_assign(rs3,1);
      _dh_vector_sub_assign(rs4,2);

/******************************* direction +1 *********************************/

      sp=pk+(*(piup++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sp).c1,1);
      _dhs_vector_load((*sp).c4,2);

      _dh_vector_i_add(1,1,2);

      _dhs_vector_load((*sp).c2,1);
      _dhs_vector_load((*sp).c3,2);

      _dh_vector_i_add(2,1,2);

      _dh_su3_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_i_sub_assign(rs3,2);
      _dh_vector_i_sub_assign(rs4,1);

/******************************* direction -1 *********************************/

      sm=pk+(*(pidn++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sm).c1,1);
      _dhs_vector_load((*sm).c4,2);

      _dh_vector_i_sub(1,1,2);

      _dhs_vector_load((*sm).c2,1);
      _dhs_vector_load((*sm).c3,2);

      _dh_vector_i_sub(2,1,2);

      _dh_su3_inverse_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_i_add_assign(rs3,2);
      _dh_vector_i_add_assign(rs4,1);

/******************************* direction +2 *********************************/

      sp=pk+(*(piup++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sp).c1,1);
      _dhs_vector_load((*sp).c4,2);

      _dh_vector_add(1,1,2);

      _dhs_vector_load((*sp).c2,1);
      _dhs_vector_load((*sp).c3,2);

      _dh_vector_sub(2,1,2);

      _dh_su3_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_sub_assign(rs3,2);
      _dh_vector_add_assign(rs4,1);

/******************************* direction -2 *********************************/

      sm=pk+(*(pidn++));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sm).c1,1);
      _dhs_vector_load((*sm).c4,2);

      _dh_vector_sub(1,1,2);

      _dhs_vector_load((*sm).c2,1);
      _dhs_vector_load((*sm).c3,2);

      _dh_vector_add(2,1,2);

      _dh_su3_inverse_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_add_assign(rs3,2);
      _dh_vector_sub_assign(rs4,1);

/******************************* direction +3 *********************************/

      sp=pk+(*(piup));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sp).c1,1);
      _dhs_vector_load((*sp).c3,2);

      _dh_vector_i_add(1,1,2);

      _dhs_vector_load((*sp).c2,1);
      _dhs_vector_load((*sp).c4,2);

      _dh_vector_i_sub(2,1,2);

      _dh_su3_multiply(1,2,1);

      u+=1;

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_i_sub_assign(rs3,1);
      _dh_vector_i_add_assign(rs4,2);

/******************************* direction -3 *********************************/

      sm=pk+(*(pidn));

      _dhs_su3_load(*u,1);

      _dhs_vector_load((*sm).c1,1);
      _dhs_vector_load((*sm).c3,2);

      _dh_vector_i_sub(1,1,2);

      _dhs_vector_load((*sm).c2,1);
      _dhs_vector_load((*sm).c4,2);

      _dh_vector_i_add(2,1,2);

      _dh_su3_inverse_multiply(1,2,1);

      _dh_vector_add_assign(rs1,1);
      _dh_vector_add_assign(rs2,2);
      _dh_vector_i_add_assign(rs3,1);
      _dh_vector_i_sub_assign(rs4,2);

      _dh_vector_mul_assign(rs1,-coe);
      _dh_vector_mul_assign(rs2,-coe);
      _dh_vector_mul_assign(rs3,coe);
      _dh_vector_mul_assign(rs4,coe);

}


static void qeo(int *piup,int *pidn,su3 *u,spinor *pl)
{
   spinor *sp,*sm;

#pragma disjoint(*u,*sp,*sm)
   __alignx(16,sp);
   __alignx(16,sm);
   __alignx(16,u);

   _dh_vector_declare(1);
   _dh_vector_declare(2);
   _dh_vector_declare1(s1);
   _dh_vector_declare1(s2);
   _dh_su3_declare(1);

/******************************* direction +0 *********************************/

      sp=pl+(*(piup++));

      _dh_vector_sub(1,rs1,rs3);
      _dh_vector_sub(2,rs2,rs4);

      _dhs_su3_load(*u,1);

      _dh_su3_inverse_multiply(1,2,1);

      _dhs_vector_load((*sp).c1,s1);
      _dhs_vector_load((*sp).c3,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_sub_assign(s2,1);

      _dhs_vector_store((*sp).c1,s1);
      _dhs_vector_store((*sp).c3,s2);

      _dhs_vector_load((*sp).c2,s1);
      _dhs_vector_load((*sp).c4,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_sub_assign(s2,2);

      _dhs_vector_store((*sp).c2,s1);
      _dhs_vector_store((*sp).c4,s2);

/******************************* direction -0 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _dh_vector_add(1,rs1,rs3);
      _dh_vector_add(2,rs2,rs4);

      _dhs_su3_load(*u,1);

      _dh_su3_multiply(1,2,1);

      _dhs_vector_load((*sm).c1,s1);
      _dhs_vector_load((*sm).c3,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_add_assign(s2,1);

      _dhs_vector_store((*sm).c1,s1);
      _dhs_vector_store((*sm).c3,s2);

      _dhs_vector_load((*sm).c2,s1);
      _dhs_vector_load((*sm).c4,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_add_assign(s2,2);

      _dhs_vector_store((*sm).c2,s1);
      _dhs_vector_store((*sm).c4,s2);

/******************************* direction +1 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _dh_vector_i_sub(1,rs1,rs4);
      _dh_vector_i_sub(2,rs2,rs3);

      _dhs_su3_load(*u,1);

      _dh_su3_inverse_multiply(1,2,1);

      _dhs_vector_load((*sp).c1,s1);
      _dhs_vector_load((*sp).c4,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_i_add_assign(s2,1);

      _dhs_vector_store((*sp).c1,s1);
      _dhs_vector_store((*sp).c4,s2);

      _dhs_vector_load((*sp).c2,s1);
      _dhs_vector_load((*sp).c3,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_i_add_assign(s2,2);

      _dhs_vector_store((*sp).c2,s1);
      _dhs_vector_store((*sp).c3,s2);

/******************************* direction -1 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _dh_vector_i_add(1,rs1,rs4);
      _dh_vector_i_add(2,rs2,rs3);

      _dhs_su3_load(*u,1);

      _dh_su3_multiply(1,2,1);

      _dhs_vector_load((*sm).c1,s1);
      _dhs_vector_load((*sm).c4,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_i_sub_assign(s2,1);

      _dhs_vector_store((*sm).c1,s1);
      _dhs_vector_store((*sm).c4,s2);

      _dhs_vector_load((*sm).c2,s1);
      _dhs_vector_load((*sm).c3,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_i_sub_assign(s2,2);

      _dhs_vector_store((*sm).c2,s1);
      _dhs_vector_store((*sm).c3,s2);

/******************************* direction +2 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _dh_vector_sub(1,rs1,rs4);
      _dh_vector_add(2,rs2,rs3);

      _dhs_su3_load(*u,1);

      _dh_su3_inverse_multiply(1,2,1);

      _dhs_vector_load((*sp).c1,s1);
      _dhs_vector_load((*sp).c4,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_sub_assign(s2,1);

      _dhs_vector_store((*sp).c1,s1);
      _dhs_vector_store((*sp).c4,s2);

      _dhs_vector_load((*sp).c2,s1);
      _dhs_vector_load((*sp).c3,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_add_assign(s2,2);

      _dhs_vector_store((*sp).c2,s1);
      _dhs_vector_store((*sp).c3,s2);

/******************************* direction -2 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _dh_vector_add(1,rs1,rs4);
      _dh_vector_sub(2,rs2,rs3);

      _dhs_su3_load(*u,1);

      _dh_su3_multiply(1,2,1);

      _dhs_vector_load((*sm).c1,s1);
      _dhs_vector_load((*sm).c4,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_add_assign(s2,1);

      _dhs_vector_store((*sm).c1,s1);
      _dhs_vector_store((*sm).c4,s2);

      _dhs_vector_load((*sm).c2,s1);
      _dhs_vector_load((*sm).c3,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_sub_assign(s2,2);

      _dhs_vector_store((*sm).c2,s1);
      _dhs_vector_store((*sm).c3,s2);

/******************************* direction +3 *********************************/

      sp=pl+(*(piup));
      u+=1;

      _dh_vector_i_sub(1,rs1,rs3);
      _dh_vector_i_add(2,rs2,rs4);

      _dhs_su3_load(*u,1);

      _dh_su3_inverse_multiply(1,2,1);

      _dhs_vector_load((*sp).c1,s1);
      _dhs_vector_load((*sp).c3,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_i_add_assign(s2,1);

      _dhs_vector_store((*sp).c1,s1);
      _dhs_vector_store((*sp).c3,s2);

      _dhs_vector_load((*sp).c2,s1);
      _dhs_vector_load((*sp).c4,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_i_sub_assign(s2,2);

      _dhs_vector_store((*sp).c2,s1);
      _dhs_vector_store((*sp).c4,s2);

/******************************* direction -3 *********************************/

      sm=pl+(*(pidn));
      u+=1;

      _dh_vector_i_add(1,rs1,rs3);
      _dh_vector_i_sub(2,rs2,rs4);

      _dhs_su3_load(*u,1);

      _dh_su3_multiply(1,2,1);

      _dhs_vector_load((*sm).c1,s1);
      _dhs_vector_load((*sm).c3,s2);

      _dh_vector_sub_assign(s1,1);
      _dh_vector_i_sub_assign(s2,1);

      _dhs_vector_store((*sm).c1,s1);
      _dhs_vector_store((*sm).c3,s2);

      _dhs_vector_load((*sm).c2,s1);
      _dhs_vector_load((*sm).c4,s2);

      _dh_vector_sub_assign(s1,2);
      _dh_vector_i_add_assign(s2,2);

      _dhs_vector_store((*sm).c2,s1);
      _dhs_vector_store((*sm).c4,s2);

}

#elif (defined QPX)
#include "qpx.h"

//#if (defined VERBOSE_DEBUG)
#if 0
#define _print_dir(mu) \
    printf( "\ndir %s\n", mu )

#define _cv_print(a) \
    printf( "vecX = {%f+j%f, %f+j%f, %f+j%f}\n", a.c1.re, a.c1.im, a.c2.re, a.c2.im, a.c3.re, a.c3.im ) 

#define _cv_print2(a,b) \
    printf( "vecA = {%f+j%f, %f+j%f, %f+j%f}\n", a.c1.re, a.c1.im, a.c2.re, a.c2.im, a.c3.re, a.c3.im ); \
    printf( "vecB = {%f+j%f, %f+j%f, %f+j%f}\n", b.c1.re, b.c1.im, b.c2.re, b.c2.im, b.c3.re, b.c3.im ) 

#define _qpx_debug_vectorpair_print(p) \
    _qpx_vectorpair_print(p)

#else
#define _print_dir(mu)
#define _cv_print(a)
#define _cv_print2(a,b)
#define _qpx_debug_vectorpair_print(p)

#endif // get rid of debug output

_qpx_vectorpair_static_declare(r12);
_qpx_vectorpair_static_declare(r34);

static void qoe(int *piup,int *pidn,su3 *u,spinor *pk)
{
   spinor *sp,*sm;

#pragma disjoint(*u,*sp,*sm)
   __alignx(16,sp);
   __alignx(16,sm);
   __alignx(16,u);

    _qpx_su3_declare(cm);
    _qpx_vectorpair_declare(p1);
    _qpx_vectorpair_declare(p2);

/******************************* direction +0 *********************************/

      sp=pk+(*(piup++));

      _print_dir("+0");

      //_qpx_vectorpair_au_load(p1,(*sp).c1,(*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load(p2,(*sp).c3,(*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(p1,p2,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_debug_vectorpair_print( p1 );
      _qpx_debug_vectorpair_print( p2 );

      _qpx_vectorpair_add( p1, p1, p2 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( r12, cm, p1 );
      _qpx_vectorpair_assign( r34, r12 );
      
      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction -0 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _print_dir("-0");

      //_qpx_vectorpair_au_load( p1, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p2, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(p1,p2,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( p1, p1, p2 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_sub( r34, r34, p2 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction +1 *********************************/

      sp=pk+(*(piup++));
      u+=1;

      _print_dir("+1");

      //_qpx_vectorpair_au_load( p1, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_ua_load( p2, (*sp).c4, (*sp).c3, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor43_load(p1,p2,(*sp),cm_1_0,cm_1_1,cm_1_2,cm_2_0);

      _qpx_vectorpair_i_add( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1+i*s4, s2+i*s3 }
      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1+i*s4, s2+i*s3 }
      // r34 -= i*U*{ s2+i*s3, s1+i*s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_i_xsub( r34, r34, p2, cm_1, cm_2_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction -1 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _print_dir("-1");

      //_qpx_vectorpair_au_load( p1, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_ua_load( p2, (*sm).c4, (*sm).c3, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor43_load(p1,p2,(*sm),cm_1_0,cm_1_1,cm_1_2,cm_2_0);

      _qpx_vectorpair_i_sub( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1-i*s4, s2-i*s3 }
      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1-i*s4, s2-i*s3 }
      // r34 += i*U*{ s2-i*s3, s1-i*s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_i_xadd( r34, r34, p2, cm_1, cm_2_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction +2 *********************************/

      sp=pk+(*(piup++));
      u+=1;

      _print_dir("+2");

      //_qpx_vectorpair_au_load( p1, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_ua_load( p2, (*sp).c4, (*sp).c3, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor43_load(p1,p2,(*sp),cm_1_0,cm_1_1,cm_1_2,cm_2_0);

      _qpx_vectorpair_1add2sub( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1+s4, s2-s3 }
      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1+s4, s2-s3 }
      // r34 += U*{ -1 * s2-s3, s1+s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_x1sub2add( r34, r34, p2, cm_1, cm_2_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction -2 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _print_dir("-2");

      //_qpx_vectorpair_au_load( p1, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_ua_load( p2, (*sm).c4, (*sm).c3, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor43_load(p1,p2,(*sm),cm_1_0,cm_1_1,cm_1_2,cm_2_0);

      _qpx_vectorpair_1sub2add( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1-s4, s2+s3 }
      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1-s4, s2+s3 }
      // r34 += U*{ s2+s3, -1 * s1-s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_x1add2sub( r34, r34, p2, cm_1, cm_2_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction +3 *********************************/

      sp=pk+(*(piup));
      u+=1;

      _print_dir("+3");

      //_qpx_vectorpair_au_load( p1, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p2, (*sp).c3, (*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(p1,p2,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_i_1add2sub( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1+i*s3, s2-i*s4 }
      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1+i*s3, s2-i*s4 }
      // r34 += i*U*{ -1 * s1+i*s3, s2-i*s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_i_1sub2add( r34, r34, p2, cm_1_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

/******************************* direction -3 *********************************/

      sm=pk+(*(pidn));
      u+=1;

      _print_dir("-3");

      //_qpx_vectorpair_au_load( p1, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p2, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(p1,p2,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_i_1sub2add( p1, p1, p2, cm_1_0 );
      _qpx_debug_vectorpair_print( p1 );

      // p2 = U*{ s1-i*s3, s2+i*s4 }
      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      // r12 += U*{ s1-i*s3, s2+i*s4 }
      // r34 += i*U*{ s1-i*s3, -1 * s2+i*s4 }
      _qpx_vectorpair_add( r12, r12, p2 );
      _qpx_vectorpair_i_1add2sub( r34, r34, p2, cm_1_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );

      cm_1_0 = VEC_LOAD_S( 0l, &coe );
      _qpx_vectorpair_mul( r34, r34, cm_1_0 );
      cm_1_0 = vec_neg( cm_1_0 );
      _qpx_vectorpair_mul( r12, r12, cm_1_0 );

      _qpx_debug_vectorpair_print( r12 );
      _qpx_debug_vectorpair_print( r34 );
}

static void qeo(int *piup,int *pidn,su3 *u,spinor *pl)
{
   spinor *sp,*sm;

#pragma disjoint(*u,*sp,*sm)
   __alignx(16,sp);
   __alignx(16,sm);
   __alignx(16,u);

    _qpx_su3_declare(cm);
    _qpx_vectorpair_declare(p1);
    _qpx_vectorpair_declare(p2);

    _qpx_debug_vectorpair_print( r12 );
    _qpx_debug_vectorpair_print( r34 );

/******************************* direction +0 *********************************/

      sp=pl+(*(piup++));

      _print_dir("+0");

      _qpx_vectorpair_sub( p1, r12, r34 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sp).c3, (*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_sub( p1, p1, p2 );

      //_qpx_vectorpair_au_store( (*sp).c1, (*sp).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sp).c3, (*sp).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sp), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sp).c1, (*sp).c2 );
      _cv_print2( (*sp).c3, (*sp).c4 );

/******************************* direction -0 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _print_dir("-0");

      _qpx_vectorpair_add( p1, r12, r34 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      _cv_print2( (*sm).c1, (*sm).c2 );
      _cv_print2( (*sm).c3, (*sm).c4 );

      //_qpx_vectorpair_au_load( cm_0, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_add( p1, p1, p2 );

      //_qpx_vectorpair_au_store( (*sm).c1, (*sm).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sm).c3, (*sm).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sm), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sm).c1, (*sm).c2 );
      _cv_print2( (*sm).c3, (*sm).c4 );

/******************************* direction +1 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _print_dir("+1");

      _qpx_vectorpair_i_xsub( p1, r12, r34, cm_1, cm_2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sp).c3, (*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_i_xadd( p1, p1, p2, cm_1, cm_2_0 );

      //_qpx_vectorpair_au_store( (*sp).c1, (*sp).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sp).c3, (*sp).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sp), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sp).c1, (*sp).c2 );
      _cv_print2( (*sp).c3, (*sp).c4 );

/******************************* direction -1 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _print_dir("-1");

      _qpx_vectorpair_i_xadd( p1, r12, r34, cm_1, cm_2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_i_xsub( p1, p1, p2, cm_1, cm_2_0 );

      //_qpx_vectorpair_au_store( (*sm).c1, (*sm).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sm).c3, (*sm).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sm), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sm).c1 , (*sm).c2 );
      _cv_print2( (*sm).c3 , (*sm).c4 );

/******************************* direction +2 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _print_dir("+2");

      _qpx_vectorpair_x1sub2add( p1, r12, r34, cm_1, cm_2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sp).c3, (*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_x1add2sub( p1, p1, p2, cm_1, cm_2_0 );

      //_qpx_vectorpair_au_store( (*sp).c1, (*sp).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sp).c3, (*sp).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sp), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sp).c1, (*sp).c2 );
      _cv_print2( (*sp).c3, (*sp).c4 );

/******************************* direction -2 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _print_dir("-2");

      _qpx_vectorpair_x1add2sub( p1, r12, r34, cm_1, cm_2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_x1sub2add( p1, p1, p2, cm_1, cm_2_0 );

      //_qpx_vectorpair_au_store( (*sm).c1, (*sm).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sm).c3, (*sm).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sm), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sm).c1, (*sm).c2 );
      _cv_print2( (*sm).c3, (*sm).c4 );

/******************************* direction +3 *********************************/

      sp=pl+(*(piup));
      u+=1;

      _print_dir("+3");

      _qpx_vectorpair_i_1sub2add( p1, r12, r34, p2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_a_load( cm, (*u), p2_0 );
      _qpx_su3_inv_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sp).c1, (*sp).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sp).c3, (*sp).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sp),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_i_1add2sub( p1, p1, p2, cm_1_0 );

      //_qpx_vectorpair_au_store( (*sp).c1, (*sp).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sp).c3, (*sp).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sp), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sp).c1, (*sp).c2 );
      _cv_print2( (*sp).c3, (*sp).c4 );

/******************************* direction -3 *********************************/

      sm=pl+(*(pidn));
      u+=1;

      _print_dir("-3");

      _qpx_vectorpair_i_1add2sub( p1, r12, r34, p2_0 );
      _qpx_debug_vectorpair_print( p1 );

      _qpx_su3_u_load( cm, (*u), p2_0 );
      _qpx_su3_multiply( p2, cm, p1 );
      _qpx_debug_vectorpair_print( p2 );

      //_qpx_vectorpair_au_load( cm_0, (*sm).c1, (*sm).c2, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_load( p1, (*sm).c3, (*sm).c4, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_load(cm_0,p1,(*sm),cm_1_0,cm_1_1,cm_1_2 );

      _qpx_vectorpair_sub( cm_0, cm_0, p2 );
      _qpx_vectorpair_i_1sub2add( p1, p1, p2, cm_1_0 );

      //_qpx_vectorpair_au_store( (*sm).c1, (*sm).c2, cm_0, cm_1_0, cm_1_1 );
      //_qpx_vectorpair_au_store( (*sm).c3, (*sm).c4, p1, cm_1_0, cm_1_1 );
      _qpx_vectorpair_spinor34_store( (*sm), cm_0, p1, cm_1_0, cm_1_1, cm_1_2 );

      _cv_print2( (*sm).c1, (*sm).c2 );
      _cv_print2( (*sm).c3, (*sm).c4 );
}

#else

#define _vector_mul_assign(r,c) \
   (r).c1.re*=(c); \
   (r).c1.im*=(c); \
   (r).c2.re*=(c); \
   (r).c2.im*=(c); \
   (r).c3.re*=(c); \
   (r).c3.im*=(c)

static spinor rs;


static void qoe(int *piup,int *pidn,su3 *u,spinor *pk)
{
   spinor *sp,*sm;
   su3_vector psi,chi;

/******************************* direction +0 *********************************/

      sp=pk+(*(piup++));

      _vector_add(psi,(*sp).c1,(*sp).c3);
      _su3_multiply(rs.c1,*u,psi);
      rs.c3=rs.c1;

      _vector_add(psi,(*sp).c2,(*sp).c4);
      _su3_multiply(rs.c2,*u,psi);
      rs.c4=rs.c2;

/******************************* direction -0 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _vector_sub(psi,(*sm).c1,(*sm).c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_sub_assign(rs.c3,chi);

      _vector_sub(psi,(*sm).c2,(*sm).c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_sub_assign(rs.c4,chi);

/******************************* direction +1 *********************************/

      sp=pk+(*(piup++));
      u+=1;

      _vector_i_add(psi,(*sp).c1,(*sp).c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_i_sub_assign(rs.c4,chi);

      _vector_i_add(psi,(*sp).c2,(*sp).c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_i_sub_assign(rs.c3,chi);

/******************************* direction -1 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _vector_i_sub(psi,(*sm).c1,(*sm).c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_i_add_assign(rs.c4,chi);

      _vector_i_sub(psi,(*sm).c2,(*sm).c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_i_add_assign(rs.c3,chi);

/******************************* direction +2 *********************************/

      sp=pk+(*(piup++));
      u+=1;

      _vector_add(psi,(*sp).c1,(*sp).c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_add_assign(rs.c4,chi);

      _vector_sub(psi,(*sp).c2,(*sp).c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_sub_assign(rs.c3,chi);

/******************************* direction -2 *********************************/

      sm=pk+(*(pidn++));
      u+=1;

      _vector_sub(psi,(*sm).c1,(*sm).c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_sub_assign(rs.c4,chi);

      _vector_add(psi,(*sm).c2,(*sm).c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_add_assign(rs.c3,chi);

/******************************* direction +3 *********************************/

      sp=pk+(*(piup));
      u+=1;

      _vector_i_add(psi,(*sp).c1,(*sp).c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_i_sub_assign(rs.c3,chi);

      _vector_i_sub(psi,(*sp).c2,(*sp).c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_i_add_assign(rs.c4,chi);

/******************************* direction -3 *********************************/

      sm=pk+(*(pidn));
      u+=1;

      _vector_i_sub(psi,(*sm).c1,(*sm).c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c1,chi);
      _vector_i_add_assign(rs.c3,chi);

      _vector_i_add(psi,(*sm).c2,(*sm).c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign(rs.c2,chi);
      _vector_i_sub_assign(rs.c4,chi);

      _vector_mul_assign(rs.c1,-coe);
      _vector_mul_assign(rs.c2,-coe);
      _vector_mul_assign(rs.c3,coe);
      _vector_mul_assign(rs.c4,coe);
}


static void qeo(int *piup,int *pidn,su3 *u,spinor *pl)
{
   spinor *sp,*sm;
   su3_vector psi,chi;

/******************************* direction +0 *********************************/

      sp=pl+(*(piup++));

      _vector_sub(psi,rs.c1,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c1,chi);
      _vector_sub_assign((*sp).c3,chi);

      _vector_sub(psi,rs.c2,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c2,chi);
      _vector_sub_assign((*sp).c4,chi);

/******************************* direction -0 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _vector_add(psi,rs.c1,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c1,chi);
      _vector_add_assign((*sm).c3,chi);

      _vector_add(psi,rs.c2,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c2,chi);
      _vector_add_assign((*sm).c4,chi);

/******************************* direction +1 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _vector_i_sub(psi,rs.c1,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c1,chi);
      _vector_i_add_assign((*sp).c4,chi);

      _vector_i_sub(psi,rs.c2,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c2,chi);
      _vector_i_add_assign((*sp).c3,chi);

/******************************* direction -1 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _vector_i_add(psi,rs.c1,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c1,chi);
      _vector_i_sub_assign((*sm).c4,chi);

      _vector_i_add(psi,rs.c2,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c2,chi);
      _vector_i_sub_assign((*sm).c3,chi);

/******************************* direction +2 *********************************/

      sp=pl+(*(piup++));
      u+=1;

      _vector_sub(psi,rs.c1,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c1,chi);
      _vector_sub_assign((*sp).c4,chi);

      _vector_add(psi,rs.c2,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c2,chi);
      _vector_add_assign((*sp).c3,chi);

/******************************* direction -2 *********************************/

      sm=pl+(*(pidn++));
      u+=1;

      _vector_add(psi,rs.c1,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c1,chi);
      _vector_add_assign((*sm).c4,chi);

      _vector_sub(psi,rs.c2,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c2,chi);
      _vector_sub_assign((*sm).c3,chi);

/******************************* direction +3 *********************************/

      sp=pl+(*(piup));
      u+=1;

      _vector_i_sub(psi,rs.c1,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c1,chi);
      _vector_i_add_assign((*sp).c3,chi);

      _vector_i_add(psi,rs.c2,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_sub_assign((*sp).c2,chi);
      _vector_i_sub_assign((*sp).c4,chi);

/******************************* direction -3 *********************************/

      sm=pl+(*(pidn));
      u+=1;

      _vector_i_add(psi,rs.c1,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c1,chi);
      _vector_i_sub_assign((*sm).c3,chi);

      _vector_i_sub(psi,rs.c2,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_sub_assign((*sm).c2,chi);
      _vector_i_add_assign((*sm).c4,chi);
}

#endif


void Qhat(int k,int l)
{
   int *piup,*pidn,iprms[2];
   su3 *u,*um;
   pauli *m;
   spinor *pk,*pl;

#if (defined DH)
#pragma disjoint(*m,*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pk);
   __alignx(16,pl);

   _dh_weyl_vector_declare(1);
   _dh_pauli_matrix_declare(1);
#elif (defined QPX)
#pragma disjoint(*m,*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pk);
   __alignx(16,pl);

   _qpx_pauli_declare(pm);
   _qpx_vectorpair_declare(pm_x);
   _qpx_vector_declare(perm);
   _qpx_vector_declare(work);
#endif

   iprms[0]=k;
   iprms[1]=l;

   MPI_Bcast(iprms,2,MPI_INT,0,MPI_COMM_WORLD);

   error((iprms[0]!=k)||(iprms[1]!=l),1,
         "Qhat [Qhat.c]","Parameters k,l are not global");

   error_root((k<0)||(k>=no_s)||(l<0)||(l>=no_s)||(k==l),1,
              "Qhat [Qhat.c]","Improper arguments");

   pk=ps[k][0];
   pl=ps[l][0];

   apply_sw(VOLUME/2,sw,pk,pl);

   if (NPROC>1) {
     set_s2zero(BNDRY/2,pl+VOLUME);
     cps_int_bnd(k);
   }

   coe=-0.25f;
   piup=&iup[VOLUME/2][0];
   pidn=&idn[VOLUME/2][0];
   m=sw+VOLUME;
   u=pu[VOLUME/2][0];
   um=u+4*VOLUME;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _prefetch_pauli_dble(m);
#endif
      qoe(piup,pidn,u,pk);
#if (defined DH)
      _dh_weyl_vector_assign(1,rs1,rs2);
      _dhs_pauli_load(*m,1);
      _dh_mul_pauli(1,1);
      m+=1;
      _dh_vector_weyl_assign(rs1,rs2,1);
      _dh_weyl_vector_assign(1,rs3,rs4);
      _dhs_pauli_load(*m,1);
      m-=1;
      _dh_mul_pauli(1,1);
      _dh_vector_weyl_assign(rs3,rs4,1);
#elif (defined QPX)
      _qpx_pauli_load_all(pm,(*m));
      m+=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,r12, perm,work);
      _qpx_vectorpair_assign( r12, pm_x );

      _qpx_pauli_load_all(pm,(*m));
      m-=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,r34, perm,work);
      _qpx_vectorpair_assign( r34, pm_x );
#else
      mul_pauli(m,(weyl*)(&rs.c1),(weyl*)(&rs.c1));
      mul_pauli(m+1,(weyl*)(&rs.c1)+1,(weyl*)(&rs.c1)+1);
#endif
      qeo(piup,pidn,u,pl);

      piup+=4;
      pidn+=4;
      m+=2;
   }

   if (NPROC>1) {
     cps_ext_bnd(l);
   }
}


void Qoe(int k,int l)
{
   int *piup,*pidn,iprms[2];
   su3 *u,*um;
   spinor *pk,*pl;

#if (defined DH)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
#elif (defined QPX)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
   _qpx_vector_declare(work3);
#endif

   iprms[0]=k;
   iprms[1]=l;

   MPI_Bcast(iprms,2,MPI_INT,0,MPI_COMM_WORLD);

   error((iprms[0]!=k)||(iprms[1]!=l),1,
         "Qoe [Qhat.c]","Parameters k,l are not global");

   error_root((k<0)||(k>=no_s)||(l<0)||(l>=no_s),1,
              "Qoe [Qhat.c]","Improper arguments");

   pk=ps[k][0];
   pl=ps[l][VOLUME/2];

if (NPROC>1)
   cps_int_bnd(k);


   coe=0.5f;
   piup=&iup[VOLUME/2][0];
   pidn=&idn[VOLUME/2][0];
   u=pu[VOLUME/2][0];
   um=u+4*VOLUME;

   for (;u<um;u+=8)
   {
      qoe(piup,pidn,u,pk);

#if (defined SSE)
      _sse_spinor_load(rs);
      _sse_spinor_store(*pl);
#elif (defined DH)
      _dhs_vector_store((*pl).c1,rs1);
      _dhs_vector_store((*pl).c2,rs2);
      _dhs_vector_store((*pl).c3,rs3);
      _dhs_vector_store((*pl).c4,rs4);
#elif (defined QPX)
      //_qpx_vectorpair_au_store( (*pl).c1, (*pl).c2, r12, work1, work2 );
      //_qpx_vectorpair_au_store( (*pl).c3, (*pl).c4, r34, work1, work2 );
      _qpx_vectorpair_spinor34_store( (*pl), r12, r34, work1, work2, work3 );
#else
      *pl=rs;
#endif

      piup+=4;
      pidn+=4;
      pl+=1;
   }
}


void Qeo(int k,int l)
{
   int *piup,*pidn,iprms[2];
   su3 *u,*um;
   spinor *pk,*pl;

#if (defined DH)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
#elif (defined QPX)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
   _qpx_vector_declare(work3);
#endif

   iprms[0]=k;
   iprms[1]=l;

   MPI_Bcast(iprms,2,MPI_INT,0,MPI_COMM_WORLD);

   error((iprms[0]!=k)||(iprms[1]!=l),1,
         "Qeo [Qhat.c]","Parameters k,l are not global");

   error_root((k<0)||(k>=no_s)||(l<0)||(l>=no_s),1,
              "Qeo [Qhat.c]","Improper arguments");

   pk=ps[k][VOLUME/2];
   pl=ps[l][0];

if (NPROC>1)
   set_s2zero(BNDRY/2,pl+VOLUME);


   ceo=-0.5f;
   piup=iup[VOLUME/2];
   pidn=idn[VOLUME/2];
   u=pu[VOLUME/2][0];
   um=u+4*VOLUME;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _load_cst(ceo);
      _sse_spinor_load(*pk);
      _mul_cst();
      _mul_cst_up();
      _sse_spinor_store(rs);
#elif (defined DH)
      _dhs_vector_load((*pk).c1,rs1);
      _dhs_vector_load((*pk).c2,rs2);
      _dhs_vector_load((*pk).c3,rs3);
      _dhs_vector_load((*pk).c4,rs4);

      _dh_vector_mul_assign(rs1,ceo);
      _dh_vector_mul_assign(rs2,ceo);
      _dh_vector_mul_assign(rs3,ceo);
      _dh_vector_mul_assign(rs4,ceo);
#elif (defined QPX)
      //_qpx_vectorpair_au_load(r12,(*pk).c1,(*pk).c2, work1, work2 );
      //_qpx_vectorpair_au_load(r34,(*pk).c3,(*pk).c4, work1, work2 );
      _qpx_vectorpair_spinor34_load( r12, r34, (*pk), work1, work2, work3 );

      work1 = VEC_LOAD_S( 0l, &ceo );
      _qpx_vectorpair_mul( r12, r12, work1 );
      _qpx_vectorpair_mul( r34, r34, work1 );
#else
      _vector_mul(rs.c1,ceo,(*pk).c1);
      _vector_mul(rs.c2,ceo,(*pk).c2);
      _vector_mul(rs.c3,ceo,(*pk).c3);
      _vector_mul(rs.c4,ceo,(*pk).c4);
#endif

      qeo(piup,pidn,u,pl);

      piup+=4;
      pidn+=4;
      pk+=1;
   }

if (NPROC>1)
   cps_ext_bnd(l);

}


void Qnohat(int k,int l)
{
   int *piup,*pidn,iprms[2];
   su3 *u,*um;
   pauli *m;
   spinor *pke,*pko,*ple,*plo;

#if (defined DH)
#pragma disjoint(*m,*u,*pke,*ple,*pko,*plo)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pke);
   __alignx(16,ple);
   __alignx(16,pko);
   __alignx(16,plo);

   _dh_weyl_vector_declare(1);
   _dh_pauli_matrix_declare(1);
   _dh_vector_declare1(s1);
   _dh_vector_declare1(s2);
#elif (defined QPX)
#pragma disjoint(*m,*u,*pke,*ple,*pko,*plo)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pke);
   __alignx(16,ple);
   __alignx(16,pko);
   __alignx(16,plo);

   _qpx_pauli_declare(pm);
   _qpx_vectorpair_declare(pm_x);
   _qpx_vectorpair_declare(x);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
#endif

   iprms[0]=k;
   iprms[1]=l;

   MPI_Bcast(iprms,2,MPI_INT,0,MPI_COMM_WORLD);

   error((iprms[0]!=k)||(iprms[1]!=l),1,
         "Qnohat [Qhat.c]","Parameters k,l are not global");

   error_root((k<0)||(k>=no_s)||(l<0)||(l>=no_s)||(k==l),1,
              "Qnohat [Qhat.c]","Improper arguments");

   pke=ps[k][0];
   pko=ps[k][VOLUME/2];
   ple=ps[l][0];
   plo=ps[l][VOLUME/2];

   apply_sw(VOLUME/2,sw,pke,ple);

   if (NPROC>1) {
     set_s2zero(BNDRY/2,ple+VOLUME);
     cps_int_bnd(k);
   }

   coe=0.5f;
   ceo=0.5f;

   piup=iup[VOLUME/2];
   pidn=idn[VOLUME/2];
   m=sw+VOLUME;
   u=pu[VOLUME/2][0];
   um=u+4*VOLUME;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _prefetch_pauli_dble(m);
#endif
      qoe(piup,pidn,u,pke);

#if (defined QPX)
      //_qpx_vectorpair_au_load( x, (*pko).c1, (*pko).c2, work1, work2 );
      _qpx_vectorpair_weyl_load( x, (*pko), work1, work2 );
      _qpx_pauli_load_all(pm,(*m));
      m+=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,x, work1, work2 );
      _qpx_vectorpair_add( pm_x, pm_x, r12 );
      //_qpx_vectorpair_au_store( (*plo).c1, (*plo).c2, pm_x, work1, work2 );
      _qpx_vectorpair_weyl_store( (*plo), pm_x, x, work1 );

      //_qpx_vectorpair_au_load( x, (*pko).c3, (*pko).c4, work1, work2 );
      _qpx_vectorpair_weyl34_load( x, (*pko), work1, work2 );
      _qpx_pauli_load_all(pm,(*m));
      m-=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,x, work1, work2 );
      _qpx_vectorpair_add( pm_x, pm_x, r34 );
      //_qpx_vectorpair_au_store( (*plo).c3, (*plo).c4, pm_x, work1, work2 );
      _qpx_vectorpair_weyl34_store( (*plo), pm_x, x, work1 );
#else
      mul_pauli(m,(weyl*)(pko),(weyl*)(plo));
      mul_pauli(m+1,(weyl*)(pko)+1,(weyl*)(plo)+1);
#endif

#if (defined SSE)
      _sse_spinor_load(*plo);      

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            :
                            :
                            "m" (rs.c1.c1.re),
                            "m" (rs.c1.c3.re),
                            "m" (rs.c2.c2.re),
                            "m" (rs.c3.c1.re),
                            "m" (rs.c3.c3.re),
                            "m" (rs.c4.c2.re)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      _sse_spinor_store(*plo);      

      _load_cst(ceo);
      _sse_spinor_load(*pko);      
      _mul_cst();
      _mul_cst_up();
      _sse_spinor_store(rs);
#elif (defined DH)
      _dhs_vector_load((*plo).c1,s1);
      _dhs_vector_load((*plo).c2,s2);
      _dh_vector_add_assign(s1,rs1);
      _dh_vector_add_assign(s2,rs2);
      _dhs_vector_store((*plo).c1,s1);
      _dhs_vector_store((*plo).c2,s2);

      _dhs_vector_load((*plo).c3,s1);
      _dhs_vector_load((*plo).c4,s2);
      _dh_vector_add_assign(s1,rs3);
      _dh_vector_add_assign(s2,rs4);
      _dhs_vector_store((*plo).c3,s1);
      _dhs_vector_store((*plo).c4,s2);

      _dhs_vector_load((*pko).c1,rs1);
      _dhs_vector_load((*pko).c2,rs2);
      _dhs_vector_load((*pko).c3,rs3);
      _dhs_vector_load((*pko).c4,rs4);

      _dh_vector_mul_assign(rs1,ceo);
      _dh_vector_mul_assign(rs2,ceo);
      _dh_vector_mul_assign(rs3,ceo);
      _dh_vector_mul_assign(rs4,ceo);
#elif (defined QPX)
      //_qpx_vectorpair_au_load( x, (*pko).c1, (*pko).c2, work1, work2 );
      //_qpx_vectorpair_au_load( pm_x, (*pko).c3, (*pko).c4, work1, work2 );
      _qpx_vectorpair_spinor34_load( x, pm_x, (*pko), work1, work2, pm_0 );

      work1 = VEC_LOAD_S( 0l, &ceo );
      _qpx_vectorpair_mul( r12, x, work1 );
      _qpx_vectorpair_mul( r34, pm_x, work1 );
#else
      _vector_add_assign((*plo).c1,rs.c1);
      _vector_add_assign((*plo).c2,rs.c2);
      _vector_add_assign((*plo).c3,rs.c3);
      _vector_add_assign((*plo).c4,rs.c4);

      _vector_mul(rs.c1,ceo,(*pko).c1);
      _vector_mul(rs.c2,ceo,(*pko).c2);
      _vector_mul(rs.c3,ceo,(*pko).c3);
      _vector_mul(rs.c4,ceo,(*pko).c4);
#endif

      qeo(piup,pidn,u,ple);

      piup+=4;
      pidn+=4;
      pko+=1;
      plo+=1;
      m+=2;
   }

if (NPROC>1)
   cps_ext_bnd(l);

}


void Qhat_blk(block_t *b,int k,int l)
{
   int *piup,*pidn;
   su3 *u,*um;
   spinor *pk,*pl;
   pauli *m;

#if (defined DH)
#pragma disjoint(*m,*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pk);
   __alignx(16,pl);

   _dh_weyl_vector_declare(1);
   _dh_pauli_matrix_declare(1);
#elif (defined QPX)
#pragma disjoint(*m,*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pk);
   __alignx(16,pl);

   _qpx_pauli_declare(pm);
   _qpx_vectorpair_declare(pm_x);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
#endif

   if (error_loc((k<0)||(k>=(*b).ns)||(l<0)||(l>=(*b).ns)||(k==l),1,
                 "Qhat_blk [Qhat.c]","Improper arguments")!=0)
      return;

   pk=(*b).s[k];
   pl=(*b).s[l];

   apply_sw((*b).volh,(*b).sw,pk,pl);

   pk[(*b).vol]=s0;
   pl[(*b).vol]=s0;

   coe=-0.25f;
   piup=(*b).iup[(*b).volh];
   pidn=(*b).idn[(*b).volh];
   m=(*b).sw+(*b).vol;
   u=(*b).u[0];
   um=u+4*(*b).vol;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _prefetch_pauli_dble(m);
#endif
      qoe(piup,pidn,u,pk);      
#if (defined DH)
      _dh_weyl_vector_assign(1,rs1,rs2);
      _dhs_pauli_load(*m,1);
      _dh_mul_pauli(1,1);
      m+=1;
      _dh_vector_weyl_assign(rs1,rs2,1);
      _dh_weyl_vector_assign(1,rs3,rs4);
      _dhs_pauli_load(*m,1);
      m-=1;
      _dh_mul_pauli(1,1);
      _dh_vector_weyl_assign(rs3,rs4,1);
#elif (defined QPX)
      _qpx_pauli_load_all(pm,(*m));
      m+=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,r12, work1,work2);
      _qpx_vectorpair_assign( r12, pm_x );

      _qpx_pauli_load_all(pm,(*m));
      m-=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,r34, work1,work2);
      _qpx_vectorpair_assign( r34, pm_x );
#else
      mul_pauli(m,(weyl*)(&rs.c1),(weyl*)(&rs.c1));
      mul_pauli(m+1,(weyl*)(&rs.c1)+1,(weyl*)(&rs.c1)+1);
#endif
      qeo(piup,pidn,u,pl);

      piup+=4;
      pidn+=4;
      m+=2;
   }
}


void Qoe_blk(block_t *b,int k,int l)
{
   int *piup,*pidn;
   su3 *u,*um;
   spinor *pk,*pl;

#if (defined DH)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
#elif (defined QPX)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
   _qpx_vector_declare(work3);
#endif

   if (error_loc((k<0)||(k>=(*b).ns)||(l<0)||(l>=(*b).ns),1,
                 "Qoe_blk [Qhat.c]","Improper arguments")!=0)
      return;

   pk=(*b).s[k];
   pl=(*b).s[l]+(*b).volh;

   pk[(*b).vol]=s0;

   coe=0.5f;
   piup=(*b).iup[(*b).volh];
   pidn=(*b).idn[(*b).volh];
   u=(*b).u[0];
   um=u+4*(*b).vol;

   for (;u<um;u+=8)
   {
      qoe(piup,pidn,u,pk);

#if (defined SSE)
      _sse_spinor_load(rs);
      _sse_spinor_store(*pl);
#elif (defined DH)
      _dhs_vector_store((*pl).c1,rs1);
      _dhs_vector_store((*pl).c2,rs2);
      _dhs_vector_store((*pl).c3,rs3);
      _dhs_vector_store((*pl).c4,rs4);
#elif (defined QPX)
      //_qpx_vectorpair_au_store( (*pl).c1, (*pl).c2, r12, work1, work2 );
      //_qpx_vectorpair_au_store( (*pl).c3, (*pl).c4, r34, work1, work2 );
      _qpx_vectorpair_spinor34_store( (*pl), r12, r34, work1, work2, work3 );
#else
      *pl=rs;
#endif

      piup+=4;
      pidn+=4;
      pl+=1;
   }
}


void Qeo_blk(block_t *b,int k,int l)
{
   int *piup,*pidn;
   su3 *u,*um;
   spinor *pk,*pl;

#if (defined DH)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
#elif (defined QPX)
#pragma disjoint(*u,*pk,*pl)
   __alignx(16,u);
   __alignx(16,pk);
   __alignx(16,pl);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
   _qpx_vector_declare(work3);
#endif

   if (error_loc((k<0)||(k>=(*b).ns)||(l<0)||(l>=(*b).ns),1,
                 "Qeo_blk [Qhat.c]","Improper arguments")!=0)
      return;

   pk=(*b).s[k]+(*b).volh;
   pl=(*b).s[l];

   pl[(*b).vol]=s0;

   ceo=-0.5f;
   piup=(*b).iup[(*b).volh];
   pidn=(*b).idn[(*b).volh];
   u=(*b).u[0];
   um=u+4*(*b).vol;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _load_cst(ceo);
      _sse_spinor_load(*pk);      
      _mul_cst();
      _mul_cst_up();
      _sse_spinor_store(rs);
#elif (defined DH)
      _dhs_vector_load((*pk).c1,rs1);
      _dhs_vector_load((*pk).c2,rs2);
      _dhs_vector_load((*pk).c3,rs3);
      _dhs_vector_load((*pk).c4,rs4);

      _dh_vector_mul_assign(rs1,ceo);
      _dh_vector_mul_assign(rs2,ceo);
      _dh_vector_mul_assign(rs3,ceo);
      _dh_vector_mul_assign(rs4,ceo);
#elif (defined QPX)
      //_qpx_vectorpair_au_load(r12,(*pk).c1,(*pk).c2, work1, work2 );
      //_qpx_vectorpair_au_load(r34,(*pk).c3,(*pk).c4, work1, work2 );
      _qpx_vectorpair_spinor34_load( r12, r34, (*pk), work1, work2, work3 );

      work1 = VEC_LOAD_S( 0l, &ceo );
      _qpx_vectorpair_mul( r12, r12, work1 );
      _qpx_vectorpair_mul( r34, r34, work1 );
#else
      _vector_mul(rs.c1,ceo,(*pk).c1);
      _vector_mul(rs.c2,ceo,(*pk).c2);
      _vector_mul(rs.c3,ceo,(*pk).c3);
      _vector_mul(rs.c4,ceo,(*pk).c4);
#endif

      qeo(piup,pidn,u,pl);

      piup+=4;
      pidn+=4;
      pk+=1;
   }
}


void Qnohat_blk(block_t *b,int k,int l)
{
   int *piup,*pidn;
   su3 *u,*um;
   spinor *pke,*pko,*ple,*plo;
   pauli *m;

#if (defined DH)
#pragma disjoint(*m,*u,*pke,*ple,*pko,*plo)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pke);
   __alignx(16,ple);
   __alignx(16,pko);
   __alignx(16,plo);

   _dh_weyl_vector_declare(1);
   _dh_pauli_matrix_declare(1);
   _dh_vector_declare1(s1);
   _dh_vector_declare1(s2);
#elif (defined QPX)
#pragma disjoint(*m,*u,*pke,*ple,*pko,*plo)
   __alignx(16,u);
   __alignx(16,m);
   __alignx(16,pke);
   __alignx(16,ple);
   __alignx(16,pko);
   __alignx(16,plo);

   _qpx_pauli_declare(pm);
   _qpx_vectorpair_declare(pm_x);
   _qpx_vectorpair_declare(x);
   _qpx_vector_declare(work1);
   _qpx_vector_declare(work2);
#endif

   if (error_loc((k<0)||(k>=(*b).ns)||(l<0)||(l>=(*b).ns)||(k==l),1,
                 "Qnohat_blk [Qhat.c]","Improper arguments")!=0)
      return;

   pke=(*b).s[k];
   pko=(*b).s[k]+(*b).volh;
   ple=(*b).s[l];
   plo=(*b).s[l]+(*b).volh;

   apply_sw((*b).volh,(*b).sw,pke,ple);

   pke[(*b).vol]=s0;
   ple[(*b).vol]=s0;

   coe=0.5f;
   ceo=0.5f;

   piup=(*b).iup[(*b).volh];
   pidn=(*b).idn[(*b).volh];
   m=(*b).sw+(*b).vol;
   u=(*b).u[0];
   um=u+4*(*b).vol;

   for (;u<um;u+=8)
   {
#if (defined SSE)
      _prefetch_pauli_dble(m);
#endif
      qoe(piup,pidn,u,pke);
#if (defined SSE)
      mul_pauli(m,(weyl*)(pko),(weyl*)(plo));
      mul_pauli(m+1,(weyl*)(pko)+1,(weyl*)(plo)+1);

      _sse_spinor_load(*plo);      

      __asm__ __volatile__ ("addps %0, %%xmm0 \n\t"
                            "addps %1, %%xmm1 \n\t"
                            "addps %2, %%xmm2 \n\t"
                            "addps %3, %%xmm3 \n\t"
                            "addps %4, %%xmm4 \n\t"
                            "addps %5, %%xmm5"
                            : \
                            :
                            "m" (rs.c1.c1.re),
                            "m" (rs.c1.c3.re),
                            "m" (rs.c2.c2.re),
                            "m" (rs.c3.c1.re),
                            "m" (rs.c3.c3.re),
                            "m" (rs.c4.c2.re)
                            :
                            "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");

      _sse_spinor_store(*plo);

      _load_cst(ceo);
      _sse_spinor_load(*pko);
      _mul_cst();
      _mul_cst_up();
      _sse_spinor_store(rs);
#elif (defined DH)
      mul_pauli(m,(weyl*)(pko),(weyl*)(plo));
      mul_pauli(m+1,(weyl*)(pko)+1,(weyl*)(plo)+1);

      _dhs_vector_load((*plo).c1,s1);
      _dhs_vector_load((*plo).c2,s2);
      _dh_vector_add_assign(s1,rs1);
      _dh_vector_add_assign(s2,rs2);
      _dhs_vector_store((*plo).c1,s1);
      _dhs_vector_store((*plo).c2,s2);

      _dhs_vector_load((*plo).c3,s1);
      _dhs_vector_load((*plo).c4,s2);
      _dh_vector_add_assign(s1,rs3);
      _dh_vector_add_assign(s2,rs4);
      _dhs_vector_store((*plo).c3,s1);
      _dhs_vector_store((*plo).c4,s2);

      _dhs_vector_load((*pko).c1,rs1);
      _dhs_vector_load((*pko).c2,rs2);
      _dhs_vector_load((*pko).c3,rs3);
      _dhs_vector_load((*pko).c4,rs4);

      _dh_vector_mul_assign(rs1,ceo);
      _dh_vector_mul_assign(rs2,ceo);
      _dh_vector_mul_assign(rs3,ceo);
      _dh_vector_mul_assign(rs4,ceo);
#elif (defined QPX)
      //_qpx_vectorpair_au_load( x, (*pko).c1, (*pko).c2, work1, work2 );
      _qpx_vectorpair_weyl_load( x, (*pko), work1, work2 );
      _qpx_pauli_load_all(pm,(*m));
      m+=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,x, work1,work2);
      _qpx_vectorpair_add( pm_x, pm_x, r12 );
      //_qpx_vectorpair_au_store( (*plo).c1, (*plo).c2, pm_x, work1, work2 );
      _qpx_vectorpair_weyl_store( (*plo), pm_x, x, work1 );

      //_qpx_vectorpair_au_load( x, (*pko).c3, (*pko).c4, work1, work2 );
      _qpx_vectorpair_weyl34_load( x, (*pko), work1, work2 );
      _qpx_pauli_load_all(pm,(*m));
      m-=1;
      _qpx_pauli_multiply_unsafe(pm_x,pm,x, work1,work2);
      _qpx_vectorpair_add( pm_x, pm_x, r34 );
      //_qpx_vectorpair_au_store( (*plo).c3, (*plo).c4, pm_x, work1, work2 );
      _qpx_vectorpair_weyl34_store( (*plo), pm_x, x, work1 );


      //_qpx_vectorpair_au_load( x, (*pko).c1, (*pko).c2, work1, work2 );
      //_qpx_vectorpair_au_load( pm_x, (*pko).c3, (*pko).c4, work1, work2 );
      _qpx_vectorpair_spinor34_load( x, pm_x, (*pko), work1, work2, pm_0 );

      work1 = VEC_LOAD_S( 0l, &ceo );
      _qpx_vectorpair_mul( r12, x, work1 );
      _qpx_vectorpair_mul( r34, pm_x, work1 );
#else
      mul_pauli(m,(weyl*)(pko),(weyl*)(plo));
      mul_pauli(m+1,(weyl*)(pko)+1,(weyl*)(plo)+1);

      _vector_add_assign((*plo).c1,rs.c1);
      _vector_add_assign((*plo).c2,rs.c2);
      _vector_add_assign((*plo).c3,rs.c3);
      _vector_add_assign((*plo).c4,rs.c4);

      _vector_mul(rs.c1,ceo,(*pko).c1);
      _vector_mul(rs.c2,ceo,(*pko).c2);
      _vector_mul(rs.c3,ceo,(*pko).c3);
      _vector_mul(rs.c4,ceo,(*pko).c4);
#endif

      qeo(piup,pidn,u,ple);

      piup+=4;
      pidn+=4;
      pko+=1;
      plo+=1;
      m+=2;
   }
}

#if 0
/* -g5 * qeo */ 
static inline void gs_res_d(int *piup,int *pidn,su3 **uu,spinor *pl, spinor rs)
{
   spinor *sp,*sm;
   su3_vector psi,chi;
   su3 *u;
   
   _vector_mul(rs.c1, 0.5,rs.c1);
   _vector_mul(rs.c2, 0.5,rs.c2);
   _vector_mul(rs.c3, -0.5,rs.c3);
   _vector_mul(rs.c4, -0.5,rs.c4);
   
/******************************* direction +0 *********************************/
   u=uu[0];
      sp=pl+(*(piup++));

      _vector_sub(psi,rs.c1,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c1,chi);
      _vector_sub_assign((*sp).c3,chi);

      _vector_sub(psi,rs.c2,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c2,chi);
      _vector_sub_assign((*sp).c4,chi);

/******************************* direction -0 *********************************/

      sm=pl+(*(pidn++));
      
      u=uu[1];
      _vector_add(psi,rs.c1,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c1,chi);
      _vector_add_assign((*sm).c3,chi);

      _vector_add(psi,rs.c2,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c2,chi);
      _vector_add_assign((*sm).c4,chi);

/******************************* direction +1 *********************************/

      sp=pl+(*(piup++));
      
      u=uu[2];
      _vector_i_sub(psi,rs.c1,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c1,chi);
      _vector_i_add_assign((*sp).c4,chi);

      _vector_i_sub(psi,rs.c2,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c2,chi);
      _vector_i_add_assign((*sp).c3,chi);

/******************************* direction -1 *********************************/

      sm=pl+(*(pidn++));
      
      u=uu[3];
      _vector_i_add(psi,rs.c1,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c1,chi);
      _vector_i_sub_assign((*sm).c4,chi);

      _vector_i_add(psi,rs.c2,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c2,chi);
      _vector_i_sub_assign((*sm).c3,chi);

/******************************* direction +2 *********************************/

      sp=pl+(*(piup++));
      
      u=uu[4];
      _vector_sub(psi,rs.c1,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c1,chi);
      _vector_sub_assign((*sp).c4,chi);

      _vector_add(psi,rs.c2,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c2,chi);
      _vector_add_assign((*sp).c3,chi);

/******************************* direction -2 *********************************/

      sm=pl+(*(pidn++));
      
      u=uu[5];
      _vector_add(psi,rs.c1,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c1,chi);
      _vector_add_assign((*sm).c4,chi);

      _vector_sub(psi,rs.c2,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c2,chi);
      _vector_sub_assign((*sm).c3,chi);

/******************************* direction +3 *********************************/

      sp=pl+(*(piup));

      u=uu[6];
      _vector_i_sub(psi,rs.c1,rs.c3);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c1,chi);
      _vector_i_add_assign((*sp).c3,chi);

      _vector_i_add(psi,rs.c2,rs.c4);
      _su3_inverse_multiply(chi,*u,psi);
      _vector_add_assign((*sp).c2,chi);
      _vector_i_sub_assign((*sp).c4,chi);

/******************************* direction -3 *********************************/

      sm=pl+(*(pidn));
      u=uu[7];
      _vector_i_add(psi,rs.c1,rs.c3);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c1,chi);
      _vector_i_sub_assign((*sm).c3,chi);

      _vector_i_sub(psi,rs.c2,rs.c4);
      _su3_multiply(chi,*u,psi);
      _vector_add_assign((*sm).c2,chi);
      _vector_i_add_assign((*sm).c4,chi);
}

#ifdef SSE
static inline __attribute__((always_inline)) void sse_vaddsubass(spinor *pl, spinor rrs)
{


     float *ff;
     ff = (float*)pl;
     
     __m128 av, bv, cv, dv, ev, fv;
     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_add_ps(av, dv);
     bv = _mm_add_ps(bv, ev);
     cv = _mm_add_ps(cv, fv);
     
     ff = (float*)pl;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);

     ff+=12;


     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     ff+=12;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_sub_ps(av, dv);
     bv = _mm_sub_ps(bv, ev);
     cv = _mm_sub_ps(cv, fv);
     
     ff = (float*)pl;
     
     ff+=12;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);
}

#endif

static const su3 u0={{{{0.0f}}}};
/*
static su3 unitysu3={{{{0.0f}}}}; 
//float d, e, f, rn, *pf, *pf1;
//complex z;

unitysu3.c11.re=1.0f;
unitysu3.c22.re=1.0f;
unitysu3.c33.re=1.0f;
*/

//k is source, l solution, pk is modified!
void gs_blk(block_t *b,int k,int l)
{
   int *piup,*pidn;
   su3 *u;
   spinor *pl, *pk, *pkk;
   pauli *m;
   int i,j;
   //su3 *uu[8];
   int ix;
   int do_uptr;
   spinor rrs;


   if (error_loc((k<0)||(k>=(*b).ns)||(l<0)||(l>=(*b).ns)||(k==l),1,
                 "Qnohat_blk [Qhat.c]","Improper arguments")!=0)
      return;

   
   //assign_s2s(b->vol, b->s[1], b->s[4]); //copy original residue to field 4

   //apply_sw((*b).volh,(*b).sw,pke,ple);

      
   
   pk = b->s[k];
   pk[(*b).vol]=s0;
   if(b->uu == NULL) {
     do_uptr = 1;
   }
   else {          
     do_uptr = 0;
   }
   if(do_uptr) {
     b->uu = malloc(b->vol*8*sizeof(su3*));
     //printf("alloc mem for uu block %p\n", b);
   }
 
	//for (i=0;i<16;i++) {
   for (i=0;i<b->vol;i++) {
   
     ix = b->ipt[i];
     //printf("ix = %d\n", ix);
     piup = b->iup[ix];
     pidn = b->idn[ix];

     m = b->sw + (2*ix);

     //take the current residue
     
     
     pkk = (b->s[k] + ix);
     

     rrs = *pkk;
     mul_pauli(m,(weyl*)(&rrs.c1),(weyl*)(&rrs.c1));
     mul_pauli(m+1,(weyl*)(&rrs.c1)+1,(weyl*)(&rrs.c1)+1);

     pl = (b->s[l] + ix);
     //mul_pauli(m,(weyl*)(&pkk->c1),(weyl*)(&rs.c1));
     //mul_pauli(m+1,(weyl*)(&pkk->c1)+1,(weyl*)(&rs.c1)+1);

     //now rs contains the solution of the residual equation

     //rs = *pl;
#ifdef SSE
     /*
     float *ff;
     ff = (float*)pl;
     
     __m128 av, bv, cv, dv, ev, fv;
     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_add_ps(av, dv);
     bv = _mm_add_ps(bv, ev);
     cv = _mm_add_ps(cv, fv);
     
     ff = (float*)pl;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);

     ff+=12;


     av = _mm_load_ps(ff);
     bv = _mm_load_ps(ff+4);
     cv = _mm_load_ps(ff+8);
     
     ff = (float*)&rrs;
     
     ff+=12;
     
     dv = _mm_load_ps(ff);
     ev = _mm_load_ps(ff+4);
     fv = _mm_load_ps(ff+8);
     
     av = _mm_sub_ps(av, dv);
     bv = _mm_sub_ps(bv, ev);
     cv = _mm_sub_ps(cv, fv);
     
     ff = (float*)pl;
     
     ff+=12;
     
     _mm_store_ps(ff, av);
     _mm_store_ps(ff+4, bv);
     _mm_store_ps(ff+8, cv);
     
     */

     sse_vaddsubass(pl, rrs);
#else

     _vector_add_assign(pl->c1,rrs.c1);
     _vector_add_assign(pl->c2,rrs.c2);
     _vector_sub_assign(pl->c3,rrs.c3);
     _vector_sub_assign(pl->c4,rrs.c4);
 
#endif      
     //correct residual using rs! not pl!!
     
     *pkk=s0;
     
     if(do_uptr) {
       if(ix < b->volh) { //even point
	 for (j=0;j<4;j++) {
	   if(piup[j]==b->vol) {
	     b->uu[8*ix+2*j]=&u0;
	   }
	   else {
	     //prendi indice spinore e prendo la u che va down
	     b->uu[8*ix+2*j] = b->u[0] + (piup[j] - b->volh)*8 + 2*j + 1;
	   }
	 }
	 
	 for (j=0;j<4;j++) {
	   if(pidn[j]==b->vol) {
	     b->uu[8*ix+2*j+1]=&u0;
	   }
	   else {
	     b->uu[8*ix+2*j+1] = b->u[0] + (pidn[j] - b->volh)*8 + 2*j;
	     
	   }
	 }
       }
       
       else { //odd point
	 u=b->u[0] + 8*(ix- b->volh);
	 for (j=0;j<8;j++) {
	   b->uu[8*ix+j]=u+j;
	 }
	 
       }
       
       
     }
     
     

     gs_res_d(piup, pidn, &b->uu[8*ix], pk, rrs);
     
     
     
   }
   
      /*
      printf("block! \n");
      
      //check residue with Qhnohat_blk
      Qnohat_blk(b, l , 6); 
      mulg5(b->vol, b->s[6]);
      
      
      float rn;
      complex z;
      z.re = -1.0;
      z.im = 0.0;
      mulc_spinor_add(b->vol, b->s[4], b->s[6], z);
      mulc_spinor_add(b->vol, b->s[4], b->s[1], z);
      rn=norm_square(b->vol,0,b->s[4]);
      

      
      z.re=1.0;
      mulc_spinor_add(b->vol, b->s[4], b->s[1], z);
      float *pf, *pf1;
      if(sqrtf(rn)>0.000000001) {
	printf("rn = %e\n", rn);
	rn=norm_square(b->vol,0,b->s[1]);
	printf("rn = %e\n", rn);
	pf = (float*)(b->s[4]);
	pf1 = (float*)(b->s[1]);
	for(kk=0;kk<24*b->vol;kk++) {
	  
	  if(sqrt((pf[0] - pf1[0])*(pf[0] - pf1[0]))>0.00000000001) {
	    printf("%d %d %e %e\n", pf[0], pf1[0], kk/24, kk%24);
	    pf+=1;
	    pf1+=1;
	  }
	  //if(kk==11) printf("-------------------------\n");
	}
	//exit(0);


      }
   //sw_term();
   //assign_swd2swbgr(GCR_BLOCKS,NO_PTS); 
   //Qnohat_blk(b, l , 6);
   //f= b->s[6][4].c1.c1.re;
   //printf("%e %e %e %e\n", d, e, f, d-e);
   //printf("%e \n", rn);
   
   //exit(0);
   */


}
#endif
