/*
!===============================================================================
!
! clover_mult_ao_sse_r4.c
!
!-------------------------------------------------------------------------------
!
! Copyright (C) 2011-2013 Hinnerk Stueben
!
! This file is part of BQCD -- Berlin Quantum ChromoDynamics program
!
! BQCD is free software: you can redistribute it and/or modify
! it under the terms of the GNU General Public License as published by
! the Free Software Foundation, either version 3 of the License, or
! (at your option) any later version.
!
! BQCD is distributed in the hope that it will be useful,
! but WITHOUT ANY WARRANTY; without even the implied warranty of
! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! GNU General Public License for more details.
!
! You should have received a copy of the GNU General Public License
! along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
!
!-------------------------------------------------------------------------------
*/
# include "c_defs.h"

#ifdef USE_SIMD_SSE
/*----------------------------------------------------------------------------*/
/* definitions for SSE */

# include <xmmintrin.h>
# include <emmintrin.h>
# include <pmmintrin.h>

# define REAL_ALIGNED static __declspec(align(16)) float 

# define cLoad(b)   _mm_load_ps((float*) &(b))
# define cLoad1(b)  _mm_load1_ps((float*) &(b))
# define cStore(a,b) _mm_store_ps((float*) &(a), b)
# define cAdd(b, c) _mm_add_ps(b, c)
# define cSub(b, c) _mm_sub_ps(b, c)
# define cMul(b, c) _mm_mul_ps(b, c)
# define cAddSub(b, c) _mm_addsub_ps(b, c)

#elif USE_SIMD_QPX
/*----------------------------------------------------------------------------*/
/* definitions for QPX */

# define REAL_ALIGNED static REAL __attribute__((aligned(32))) 
# define __m128 vector4double

# define cLoad(x) vec_ld(0L, (REAL*) &(x))
# define cLoad1(x) vec_splats((double) x)

# define cStore(out, in) vec_st(in, 0L, (REAL*) &(out))

# define cAdd(a, b) vec_add(a, b)
# define cSub(a, b) vec_sub(a, b)
# define cMul(a, b) vec_mul(a, b)
# define cAddSub(a, b) vec_add(a, vec_mul(cLoad(conjgr), b))

# define _mm_shuffle_ps(a, b, c) vec_perm(a, b, c)
# define _MM_SHUFFLE(a,b,c,d) vec_gpci( (a+4) + ((b+4) << 3) + (c << 6) + (d << 9) )

#endif
/*----------------------------------------------------------------------------*/
/* general definitions */

# define cConjgRI(a) cMul(cLoad(conjg), a)
# define cConjgIR(a) cMul(cLoad(conjgr), a)

# define cLoadX(a, b, c)  STRCAT(a,rr) = cAdd(cLoad1(Re(b)), cLoad1(Re(c))); \ 
                          STRCAT(a,ii) = cAdd(cLoad1(Im(b)), cLoad1(Im(c))) 

# define cLoadXX(a, b, c) STRCAT(a,rr) = cSub(cLoad1(Re(b)), cLoad1(Re(c))); \ 
                          STRCAT(a,ii) = cSub(cLoad1(Im(b)), cLoad1(Im(c))) 

# define cLoadRI(a) cLoad(a)
# define cLoadIR(a) _mm_shuffle_ps(cLoad(a), cLoad(a), _MM_SHUFFLE(2,3,0,1)) 

# define cLoadDiagRI1(a) cMul(cLoadRI(a), cLoad(conjg1))
# define cLoadDiagIR1(a) cMul(cLoadIR(a), cLoad(conjg1r))

# define cLoadDiagRI2(a) cMul(_mm_shuffle_ps(cLoad(a), cLoad(a), _MM_SHUFFLE(0,1,3,2)), cLoad(c1110))
# define cLoadDiagIR2(a) cMul(_mm_shuffle_ps(cLoad(a), cLoad(a), _MM_SHUFFLE(1,0,2,3)), cLoad(c1101))


# define cDiag1(iaa, y1, y2, ia, x1, x2) \
y1 =          cAddSub(cMul(cLoadDiagRI1(aa(ia,iaa,i)), STRCAT(x1,rr)), cMul(cLoadDiagIR1(aa(ia,iaa,i)), STRCAT(x1,ii))); \
y1 = cAdd(y1, cAddSub(cMul(cLoadDiagRI2(aa(ia,iaa,i)), STRCAT(x2,rr)), cMul(cLoadDiagIR2(aa(ia,iaa,i)), STRCAT(x2,ii))))

# define cDiag2(iaa, y1, y2, ia, x1, x2) \
y1 = cAdd(y1, cAddSub(cMul(cLoadDiagRI1(aa(ia,iaa,i)), STRCAT(x1,rr)), cMul(cLoadDiagIR1(aa(ia,iaa,i)), STRCAT(x1,ii)))); \
y1 = cAdd(y1, cAddSub(cMul(cLoadDiagRI2(aa(ia,iaa,i)), STRCAT(x2,rr)), cMul(cLoadDiagIR2(aa(ia,iaa,i)), STRCAT(x2,ii))))



# define cLoadUpperRI1(a, b) _mm_shuffle_ps(cLoad(a), cLoad(b), _MM_SHUFFLE(1,0,1,0))
# define cLoadUpperIR1(a, b) _mm_shuffle_ps(cLoad(a), cLoad(b), _MM_SHUFFLE(0,1,0,1))

# define cLoadUpperRI2(a, b) _mm_shuffle_ps(cLoad(a), cLoad(b), _MM_SHUFFLE(3,2,3,2))
# define cLoadUpperIR2(a, b) _mm_shuffle_ps(cLoad(a), cLoad(b), _MM_SHUFFLE(2,3,2,3))


# define cUpper(iaa, y1, y2, ia1, ia2, x1, x2) \
y1 = cAdd(y1, cAddSub(cMul(cLoadUpperRI1(aa(ia1,iaa,i),aa(ia2,iaa,i)), STRCAT(x1,rr)), cMul(cLoadUpperIR1(aa(ia1,iaa,i),aa(ia2,iaa,i)), STRCAT(x1,ii)))); \
y1 = cAdd(y1, cAddSub(cMul(cLoadUpperRI2(aa(ia1,iaa,i),aa(ia2,iaa,i)), STRCAT(x2,rr)), cMul(cLoadUpperIR2(aa(ia1,iaa,i),aa(ia2,iaa,i)), STRCAT(x2,ii))))



# define cCmplxMulConjg(b, c) cAddSub(cMul(cConjgRI(cLoadRI(b)), STRCAT(c,rr)), cMul(cConjgIR(cLoadIR(b)), STRCAT(c,ii)))

# define cProdConjg(a, b, c)  a = cCmplxMulConjg(b, c)

# define cAddProdConjg(a,b,c) a = cAdd(a, cCmplxMulConjg(b, c))

# define cLower1(iaa, y1, y2, ia1, ia2, x1, x2) \
	     cProdConjg(y1, aa(ia1,iaa,i), x1); \
	  cAddProdConjg(y1, aa(ia2,iaa,i), x2)

# define cLower2(iaa, y1, y2, ia1, ia2, x1, x2) \
	  cAddProdConjg(y1, aa(ia1,iaa,i), x1); \
	  cAddProdConjg(y1, aa(ia2,iaa,i), x2)


# define cStoreX1(a, b, c) cStore(a, _mm_shuffle_ps(b, c, _MM_SHUFFLE(3,2,1,0)))
# define cStoreX2(a, b, c) cStore(a, _mm_shuffle_ps(b, c, _MM_SHUFFLE(1,0,3,2)))
# define cStoreX3(a, b, c) cStore(a, _mm_shuffle_ps(b, c, _MM_SHUFFLE(3,2,1,0)))


# define aa(s, c, i) aa_[i][c-1][s-1]
# define  x(s, c, i)  x_[i][c-1][s-1]


#ifdef PRECISION_R4
# define PRECISION _r4
#else
# define PRECISION
#endif

#ifdef USE_SIMD_SSE
# define UNDERSCORE _
#elif USE_SIMD_QPX
#define UNDERSCORE
#endif

#ifdef BLOCK
void STRCAT3(clover_mult_ao_block, PRECISION, UNDERSCORE)(COMPLEX (*aa_)[2][18], COMPLEX (*x_)[3][4], int *i1, int *i2)
#else
void STRCAT3(clover_mult_ao, PRECISION, UNDERSCORE)(COMPLEX (*aa_)[2][18], COMPLEX (*x_)[3][4], int *volh)
#endif
{
    int i;
    __m128 x1rr, x2rr, x3rr, x4rr, x5rr, x6rr;
    __m128 x1ii, x2ii, x3ii, x4ii, x5ii, x6ii;
    __m128 y1, y3, y5;
    __m128 z1, z3, z5;
    __m128 xx1rr, xx2rr, xx3rr, xx4rr, xx5rr, xx6rr;
    __m128 xx1ii, xx2ii, xx3ii, xx4ii, xx5ii, xx6ii;
    __m128 yy1, yy3, yy5;
    __m128 zz1, zz3, zz5;

    REAL_ALIGNED conjg[4]   = { 1.0, -1.0, 1.0, -1.0 };
    REAL_ALIGNED conjgr[4]  = { -1.0, 1.0, -1.0, 1.0 };

    REAL_ALIGNED conjg1[4]  = { 1.0, 0.0, 1.0, -1.0 };
    REAL_ALIGNED conjg1r[4] = { 0.0, 1.0, -1.0, 1.0 };
    REAL_ALIGNED c1101[4]   = { 1.0, 1.0, 0.0, 1.0 };
    REAL_ALIGNED c1110[4]   = { 1.0, 1.0, 1.0, 0.0 };

    TIMING_START(timing_bin_clover_mult_ao);

    aa_--;
    x_--;

#ifdef BLOCK
    for (i = *i1; i <= *i2; i++)
#else
    # pragma omp parallel for \
    private (x1rr, x2rr, x3rr, x4rr, x5rr, x6rr) \
    private (x1ii, x2ii, x3ii, x4ii, x5ii, x6ii) \
    private (y1, y3, y5) \
    private (z1, z3, z5) \
    private (xx1rr, xx2rr, xx3rr, xx4rr, xx5rr, xx6rr) \
    private (xx1ii, xx2ii, xx3ii, xx4ii, xx5ii, xx6ii) \
    private (yy1, yy3, yy5) \
    private (zz1, zz3, zz5)

    for (i = 1; i <= *volh; i++)
#endif
    {
	cLoadX(x1, x(1, 1, i), x(3, 1, i));
	cLoadX(x2, x(1, 2, i), x(3, 2, i));
	cLoadX(x3, x(1, 3, i), x(3, 3, i));
	cLoadX(x4, x(2, 1, i), x(4, 1, i));
	cLoadX(x5, x(2, 2, i), x(4, 2, i));
	cLoadX(x6, x(2, 3, i), x(4, 3, i));

        cDiag1(1, y1, y2, 1, x1, x2);
        cUpper(1, y1, y2, 3, 7, x3, x4);
        cUpper(1, y1, y2, 5, 9, x5, x6);

        cLower1(1, y3, y4, 3, 7, x1, x2);
         cDiag2(1, y3, y4, 11, x3, x4);
         cUpper(1, y3, y4, 13, 15, x5, x6);

        cLower1(1, y5, y6, 5, 9, x1, x2);
        cLower2(1, y5, y6, 13, 15, x3, x4);
         cDiag2(1, y5, y6, 17, x5, x6);

	cLoadXX(xx1, x(1, 1, i), x(3, 1, i));
	cLoadXX(xx2, x(1, 2, i), x(3, 2, i));
	cLoadXX(xx3, x(1, 3, i), x(3, 3, i));
	cLoadXX(xx4, x(2, 1, i), x(4, 1, i));
	cLoadXX(xx5, x(2, 2, i), x(4, 2, i));
	cLoadXX(xx6, x(2, 3, i), x(4, 3, i));

        cDiag1(2, yy1, yy2, 1, xx1, xx2);
        cUpper(2, yy1, yy2, 3, 7, xx3, xx4);
        cUpper(2, yy1, yy2, 5, 9, xx5, xx6);

        cLower1(2, yy3, yy4, 3, 7, xx1, xx2);
         cDiag2(2, yy3, yy4, 11, xx3, xx4);
         cUpper(2, yy3, yy4, 13, 15, xx5, xx6);

        cLower1(2, yy5, yy6, 5, 9, xx1, xx2);
        cLower2(2, yy5, yy6, 13, 15, xx3, xx4);
         cDiag2(2, yy5, yy6, 17, xx5, xx6);

	 z1 = cAdd(y1, yy1);
	 z3 = cAdd(y3, yy3);
	 z5 = cAdd(y5, yy5);

	zz1 = cSub(y1, yy1);
	zz3 = cSub(y3, yy3);
	zz5 = cSub(y5, yy5);

        cStoreX1(x(1, 1, i),  z1,  z3);
        cStoreX1(x(3, 1, i), zz1, zz3);

        cStoreX2(x(1, 2, i),  z1,  z5);
        cStoreX2(x(3, 2, i), zz1, zz5);

        cStoreX3(x(1, 3, i),  z3,  z5);
        cStoreX3(x(3, 3, i), zz3, zz5);
    }

    TIMING_STOP(timing_bin_clover_mult_ao);
}
