/*
!===============================================================================
!
! D103xyzt_r4.c
!
!-------------------------------------------------------------------------------
!
! Copyright (C) 2011-2013 Hinnerk Stueben
!
! This file is part of BQCD -- Berlin Quantum ChromoDynamics program
!
! BQCD is free software: you can redistribute it and/or modify
! it under the terms of the GNU General Public License as published by
! the Free Software Foundation, either version 3 of the License, or
! (at your option) any later version.
!
! BQCD is distributed in the hope that it will be useful,
! but WITHOUT ANY WARRANTY; without even the implied warranty of
! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! GNU General Public License for more details.
!
! You should have received a copy of the GNU General Public License
! along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
!
!-------------------------------------------------------------------------------
*/
# include "c_defs.h"

#ifdef USE_SIMD_SSE
/*------------------------------------------------------------------------*/
/* defininitions for SSE */

# include <xmmintrin.h>
# include <emmintrin.h>
# include <pmmintrin.h>

# define NAME_ STRCAT(NAME, _)
# define REAL_ALIGNED static __declspec(align(16)) float 

# define cLoad(x) _mm_load_ps((float*) &(x))
# define cLoad1(x) _mm_load1_ps((float*) &(x))

# define cStore(out, in) _mm_store_ps((float*) &(out), in)

#elif USE_SIMD_QPX
/*------------------------------------------------------------------------*/
/* defininitions for QPX */

# define NAME_ NAME
# define REAL_ALIGNED static REAL __attribute__((aligned(32))) 
# define __m128 vector4double

# define cLoad(x) vec_ld(0L, (REAL*) &(x))
# define cLoad1(x) vec_splats((double) x)

# define cStore(out, in) vec_st(in, 0L, (REAL*) &(out))

# define _mm_add_ps(a, b) vec_add(a, b)
# define _mm_sub_ps(a, b) vec_sub(a, b)
# define _mm_mul_ps(a, b) vec_mul(a, b)
# define _mm_addsub_ps(a, b) vec_add(a, vec_mul(cLoad(conjgr), b))

# define _mm_shuffle_ps(a, b, c) vec_perm(a, b, c)
# define _MM_SHUFFLE(a,b,c,d) vec_gpci( (a+4) + ((b+4) << 3) + (c << 6) + (d << 9) )

#endif
/*------------------------------------------------------------------------*/
/* generic defininitions */

# define cShuffleR(a) _mm_shuffle_ps(a, a, _MM_SHUFFLE(2,3,0,1))
# define cLoadR(x) cShuffleR(cLoad(x))
# define cStoreR(out, in) cStore(out, cShuffleR(in))

# define cAdd(a, b)     a = _mm_add_ps(a, b)
# define cSum(a, b, c)  a = _mm_add_ps(b, c)
# define cDiff(a, b, c) a = _mm_sub_ps(b, c)

#ifdef DAGGER
# define cDiff2(a, b, c) cDiff(a, c, b)
#else
# define cDiff2(a, b, c) cDiff(a, b, c)
#endif

#ifdef DAGGER
# define cDiff3(a, b, c)   a = _mm_mul_ps(cLoad(conjgr), _mm_sub_ps(b, c))
#else
# define cDiff3(a, b, c)   a = _mm_mul_ps(cLoad(conjg), _mm_sub_ps(b, c))
#endif

# define  cLoadAf(s, c)    STRCAT3(af_ri,s,c) = cLoad(a_fwd(s, c, jf)); \
                           STRCAT3(af_ir,s,c) = cLoadR(a_fwd(s, c, jf))

# define  cLoadAb(s, c)    STRCAT3(ab_ri,s,c) = cLoad(a_bwd(s, c, jb)); \
                           STRCAT3(ab_ir,s,c) = cLoadR(a_bwd(s, c, jb))

# define  cLoadU(c, d)     STRCAT3(u_rr,c,d) = cLoad1(Re(u(c, d, i))); \
                           STRCAT3(u_ii,c,d) = cLoad1(Im(u(c, d, i)))

# define cProdAfU(s, c, d) STRCAT3(bf,s,c) = _mm_addsub_ps(_mm_mul_ps(STRCAT3(af_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_ps(STRCAT3(af_ir,s,d), STRCAT3(u_ii,c,d)))
# define cProdAbU(s, c, d) STRCAT3(bb,s,c) = _mm_addsub_ps(_mm_mul_ps(STRCAT3(ab_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_ps(STRCAT3(ab_ir,s,d), STRCAT3(u_ii,c,d)))

# define cAddProdAfU(s, c, d) cAdd(STRCAT3(bf,s,c), _mm_addsub_ps(_mm_mul_ps(STRCAT3(af_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_ps(STRCAT3(af_ir,s,d), STRCAT3(u_ii,c,d))))
# define cAddProdAbU(s, c, d) cAdd(STRCAT3(bb,s,c), _mm_addsub_ps(_mm_mul_ps(STRCAT3(ab_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_ps(STRCAT3(ab_ir,s,d), STRCAT3(u_ii,c,d))))

# define cLoadOut(s, c) STRCAT3(out,s,c) = cLoad(out(s, c, i))
# define cLoadrOut(s, c) STRCAT3(out,s,c) =  cLoadR(out(s, c, i))

# define cStoreOut(s, c) cStore(out(s, c, i), STRCAT3(out,s,c))
# define cStorerOut(s, c) cStoreR(out(s, c, i), STRCAT3(out,s,c))

/*------------------------------------------------------------------------*/
#ifdef DIR_X
# define TIMING_BIN timing_bin_d_xf
#endif
#ifdef DIR_Y
# define TIMING_BIN timing_bin_d_yf
#endif
#ifdef DIR_Z
# define TIMING_BIN timing_bin_d_zf
#endif
#ifdef DIR_T
# define TIMING_BIN timing_bin_d_t
#endif

/*------------------------------------------------------------------------*/
# define   out(a, b, c)    out_[c][b-1][a-1]
# define a_fwd(a, b, c)  a_fwd_[c][b-1][a-1]
# define a_bwd(a, b, c)  a_bwd_[c][b-1][a-1]
# define     u(a, b, c)      u_[c][b-1][a-1]
# define      nn_fwd(c) nn_fwd_[c]
# define      nn_bwd(c) nn_bwd_[c]

/*------------------------------------------------------------------------*/
void NAME_          (SPINCOL_FIELD out_, 
                     SC2_FLD a_fwd_, 
                     SC2_FLD a_bwd_, 
                     REORDERED_GAUGE_FIELD u_, 
                     INTEGER nn_fwd_[], 
                     INTEGER nn_bwd_[],
#ifdef BLOCK
                     int *i1, int *i2)
#else
                     int *volh)
#endif

{
    int  i, jf, jb;
    __m128 af_ri11,af_ri12,af_ri13;
    __m128 af_ir11,af_ir12,af_ir13;
    __m128 ab_ri11,ab_ri12,ab_ri13;
    __m128 ab_ir11,ab_ir12,ab_ir13;
    __m128 u_rr11,u_rr12,u_rr13,u_rr21,u_rr22,u_rr23,u_rr31,u_rr32,u_rr33;
    __m128 u_rr41,u_rr42,u_rr43,u_rr51,u_rr52,u_rr53,u_rr61,u_rr62,u_rr63;
    __m128 u_ii11,u_ii12,u_ii13,u_ii21,u_ii22,u_ii23,u_ii31,u_ii32,u_ii33;
    __m128 u_ii41,u_ii42,u_ii43,u_ii51,u_ii52,u_ii53,u_ii61,u_ii62,u_ii63;
    __m128 bf11,bf12,bf13;
    __m128 bb14,bb15,bb16;
    __m128 out11, out31, out12, out32, out13, out33;
    __m128 tmp11, tmp31, tmp12, tmp32, tmp13, tmp33;

    REAL_ALIGNED conjg[4] = { 1.0, -1.0, 1.0, -1.0 };
    REAL_ALIGNED conjgr[4] = { -1.0, 1.0, -1.0, 1.0 };

#ifndef BLOCK
    TIMING_START(TIMING_BIN);
#endif

    out_--;
    a_fwd_--;
    a_bwd_--;
    u_--;
    nn_fwd_--;
    nn_bwd_--;

#ifdef BLOCK
    for (i = *i1; i <= *i2; i++)
#else
    #pragma omp parallel for \
    private (i, jf, jb) \
    private (af_ri11,af_ri12,af_ri13) \
    private (af_ir11,af_ir12,af_ir13) \
    private (ab_ri11,ab_ri12,ab_ri13) \
    private (ab_ir11,ab_ir12,ab_ir13) \
    private (u_rr11,u_rr12,u_rr13,u_rr21,u_rr22,u_rr23,u_rr31,u_rr32,u_rr33) \
    private (u_rr41,u_rr42,u_rr43,u_rr51,u_rr52,u_rr53,u_rr61,u_rr62,u_rr63) \
    private (u_ii11,u_ii12,u_ii13,u_ii21,u_ii22,u_ii23,u_ii31,u_ii32,u_ii33) \
    private (u_ii41,u_ii42,u_ii43,u_ii51,u_ii52,u_ii53,u_ii61,u_ii62,u_ii63) \
    private (bf11,bf12,bf13) \
    private (bb14,bb15,bb16) \
    private (out11, out31, out12, out32, out13, out33) \
    private (tmp11, tmp31, tmp12, tmp32, tmp13, tmp33)

    for (i = 1; i <= *volh; i++)
#endif
    {
	jf=nn_fwd(i);
	jb=nn_bwd(i);

        cLoadAf(1, 1);
        cLoadU(1, 1);
        cLoadU(2, 1);
        cLoadU(3, 1);
        cProdAfU(1, 1, 1);
        cProdAfU(1, 2, 1);
        cProdAfU(1, 3, 1);

        cLoadAb(1, 1);
        cLoadU(4, 1);
        cLoadU(5, 1);
        cLoadU(6, 1);
        cProdAbU(1, 4, 1);
        cProdAbU(1, 5, 1);
        cProdAbU(1, 6, 1);

        cLoadAf(1, 2);
        cLoadU(1, 2);
        cLoadU(2, 2);
        cLoadU(3, 2);
        cAddProdAfU(1, 1, 2);
        cAddProdAfU(1, 2, 2);
        cAddProdAfU(1, 3, 2);

        cLoadAb(1, 2);
        cLoadU(4, 2);
        cLoadU(5, 2);
        cLoadU(6, 2);
        cAddProdAbU(1, 4, 2);
        cAddProdAbU(1, 5, 2);
        cAddProdAbU(1, 6, 2);
	
        cLoadAf(1, 3);
        cLoadU(1, 3);
        cLoadU(2, 3);
        cLoadU(3, 3);
        cAddProdAfU(1, 1, 3);
        cAddProdAfU(1, 2, 3);
        cAddProdAfU(1, 3, 3);

        cLoadAb(1, 3);
        cLoadU(4, 3);
        cLoadU(5, 3);
        cLoadU(6, 3);
        cAddProdAbU(1, 4, 3);
        cAddProdAbU(1, 5, 3);
        cAddProdAbU(1, 6, 3);


#ifdef DIR_T
	cLoadOut(1, 1);
        cLoadOut(3, 1);
        cLoadOut(1, 2);
        cLoadOut(3, 2);
        cLoadOut(1, 3);
        cLoadOut(3, 3);
#ifdef DAGGER
        cAdd(out11, bf11);
        cAdd(out31, bb14);
        cAdd(out12, bf12);
        cAdd(out32, bb15);
        cAdd(out13, bf13);
        cAdd(out33, bb16);
#else
        cAdd(out11, bb14);
        cAdd(out31, bf11);
        cAdd(out12, bb15);
        cAdd(out32, bf12);
        cAdd(out13, bb16);
        cAdd(out33, bf13);
#endif
	cStoreOut(1, 1);
        cStoreOut(3, 1);
        cStoreOut(1, 2);
        cStoreOut(3, 2);
        cStoreOut(1, 3);
        cStoreOut(3, 3);
#endif

#ifdef DIR_X
# define cShuffleX(a) _mm_shuffle_ps(a, a, _MM_SHUFFLE(1,0,3,2))

	cSum(out11, bf11, bb14);
	cDiff3(out31, cShuffleX(bf11), cShuffleX(bb14));

	cSum(out12, bf12, bb15);
	cDiff3(out32, cShuffleX(bf12), cShuffleX(bb15));

	cSum(out13, bf13, bb16);
	cDiff3(out33, cShuffleX(bf13), cShuffleX(bb16));

	cStoreOut(1, 1);
        cStorerOut(3, 1);
        cStoreOut(1, 2);
        cStorerOut(3, 2);
        cStoreOut(1, 3);
        cStorerOut(3, 3);
#endif


#ifdef DIR_Y

# define cShuffleY(a, b) _mm_shuffle_ps(a, b, _MM_SHUFFLE(1,0,3,2))

        cSum(tmp11, bf11, bb14);
        cDiff2(tmp31, cShuffleY(bf11, bb14), cShuffleY(bb14, bf11));

        cSum(tmp12, bf12, bb15);
        cDiff2(tmp32, cShuffleY(bf12, bb15), cShuffleY(bb15, bf12));

        cSum(tmp13, bf13, bb16);
	cDiff2(tmp33, cShuffleY(bf13, bb16), cShuffleY(bb16, bf13));

	cLoadOut(1, 1);
        cLoadOut(3, 1);
        cLoadOut(1, 2);
        cLoadOut(3, 2);
        cLoadOut(1, 3);
        cLoadOut(3, 3);

        cAdd(out11, tmp11);
        cAdd(out31, tmp31);
        cAdd(out12, tmp12);
        cAdd(out32, tmp32);
        cAdd(out13, tmp13);
        cAdd(out33, tmp33);

	cStoreOut(1, 1);
        cStoreOut(3, 1);
        cStoreOut(1, 2);
        cStoreOut(3, 2);
        cStoreOut(1, 3);
        cStoreOut(3, 3);
#endif
	
#ifdef DIR_Z
# define cShuffleZ(a, b) _mm_shuffle_ps(a, b, _MM_SHUFFLE(3,2,1,0))

        cSum(tmp11, bf11, bb14);
        cDiff3(tmp31, cShuffleZ(bf11, bb14), cShuffleZ(bb14, bf11));

        cSum(tmp12, bf12, bb15);
        cDiff3(tmp32, cShuffleZ(bf12, bb15), cShuffleZ(bb15, bf12));

        cSum(tmp13, bf13, bb16);
        cDiff3(tmp33, cShuffleZ(bf13, bb16), cShuffleZ(bb16, bf13));

	cLoadOut(1, 1);
        cLoadrOut(3, 1);
        cLoadOut(1, 2);
        cLoadrOut(3, 2);
        cLoadOut(1, 3);
        cLoadrOut(3, 3);

        cAdd(out11, tmp11);
        cAdd(out31, tmp31);
        cAdd(out12, tmp12);
        cAdd(out32, tmp32);
        cAdd(out13, tmp13);
        cAdd(out33, tmp33);

	cStoreOut(1, 1);
        cStorerOut(3, 1);
        cStoreOut(1, 2);
        cStorerOut(3, 2);
        cStoreOut(1, 3);
        cStorerOut(3, 3);
#endif
    }

#ifndef BLOCK
    TIMING_STOP(TIMING_BIN);
#endif
}

# undef cCopy
# undef cCopyr
# undef cSum
# undef cDiff
# undef cDiff2
# undef cDiff2
# undef cDiff3
# undef cProd
# undef cAdd
# undef cAddSum
# undef cAddProd
# undef cLoadAf
# undef cLoadAb
# undef cLoadU
# undef cProdAfU
# undef cProdAbU
# undef cAddProdAfU
# undef cAddProdAbU
# undef cLoadOut
# undef cLoadrOut
# undef cStoreOut
# undef cStorerOut
# undef TIMING_BIN
