/*
!===============================================================================
!
! D103xyzt.c
!
!-------------------------------------------------------------------------------
!
! Copyright (C) 2011 Hinnerk Stueben
!
! This file is part of BQCD -- Berlin Quantum ChromoDynamics program
!
! BQCD is free software: you can redistribute it and/or modify
! it under the terms of the GNU General Public License as published by
! the Free Software Foundation, either version 3 of the License, or
! (at your option) any later version.
!
! BQCD is distributed in the hope that it will be useful,
! but WITHOUT ANY WARRANTY; without even the implied warranty of
! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! GNU General Public License for more details.
!
! You should have received a copy of the GNU General Public License
! along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
!
!-------------------------------------------------------------------------------
*/
# include <emmintrin.h>
# include <pmmintrin.h>
# include "c_defs.h"

# define cAdd(a, b)     a = _mm_add_pd(a, b)
# define cSum(a, b, c)  a = _mm_add_pd(b, c)
# define cDiff(a, b, c) a = _mm_sub_pd(b, c)

#ifdef DAGGER
# define cDiff2(a, b, c) cDiff(a, c, b)
#else
# define cDiff2(a, b, c) cDiff(a, b, c)
#endif

#ifdef DAGGER
# define cDiff3(a, b, c)   a = _mm_mul_pd(conjgr, _mm_sub_pd(b, c))
#else
# define cDiff3(a, b, c)   a = _mm_mul_pd(conjg, _mm_sub_pd(b, c))
#endif

# define  cLoadAf(s, c)    STRCAT3(af_ri,s,c) = _mm_load_pd((double*) &(a_fwd(s, c, jf))); \
                           STRCAT3(af_ir,s,c) = _mm_loadr_pd((double*) &(a_fwd(s, c, jf)))

# define  cLoadAb(s, c)    STRCAT3(ab_ri,s,c) = _mm_load_pd((double*) &(a_bwd(s, c, jb))); \
                           STRCAT3(ab_ir,s,c) = _mm_loadr_pd((double*) &(a_bwd(s, c, jb)))

# define  cLoadU(c, d)     STRCAT3(u_rr,c,d) = _mm_load1_pd((double*) &(Re(u(c, d, i)))); \
                           STRCAT3(u_ii,c,d) = _mm_load1_pd((double*) &(Im(u(c, d, i))))

# define cProdAfU(s, c, d) STRCAT3(bf,s,c) = _mm_addsub_pd(_mm_mul_pd(STRCAT3(af_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_pd(STRCAT3(af_ir,s,d), STRCAT3(u_ii,c,d)))
# define cProdAbU(s, c, d) STRCAT3(bb,s,c) = _mm_addsub_pd(_mm_mul_pd(STRCAT3(ab_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_pd(STRCAT3(ab_ir,s,d), STRCAT3(u_ii,c,d)))

# define cAddProdAfU(s, c, d) cAdd(STRCAT3(bf,s,c), _mm_addsub_pd(_mm_mul_pd(STRCAT3(af_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_pd(STRCAT3(af_ir,s,d), STRCAT3(u_ii,c,d))))
# define cAddProdAbU(s, c, d) cAdd(STRCAT3(bb,s,c), _mm_addsub_pd(_mm_mul_pd(STRCAT3(ab_ri,s,d), STRCAT3(u_rr,c,d)), _mm_mul_pd(STRCAT3(ab_ir,s,d), STRCAT3(u_ii,c,d))))

# define cLoadOut(s, c) STRCAT3(out,s,c) = _mm_load_pd((double*) &(out(s, c, i)))
# define cLoadrOut(s, c) STRCAT3(out,s,c) = _mm_loadr_pd((double*) &(out(s, c, i)))
# define cStoreOut(s, c) _mm_store_pd((double*) &(out(s, c, i)), STRCAT3(out,s,c))
# define cStorerOut(s, c) _mm_storer_pd((double*) &(out(s, c, i)), STRCAT3(out,s,c))


# define   out(a, b, c)    out_[c][b-1][a-1]
# define a_fwd(a, b, c)  a_fwd_[c][b-1][a-1]
# define a_bwd(a, b, c)  a_bwd_[c][b-1][a-1]
# define     u(a, b, c)      u_[c][b-1][a-1]
# define      nn_fwd(c) nn_fwd_[c]
# define      nn_bwd(c) nn_bwd_[c]

/*------------------------------------------------------------------------*/
void STRCAT(NAME, _)(SPINCOL_FIELD out_, 
                     SC2_FLD a_fwd_, 
                     SC2_FLD a_bwd_, 
                     REORDERED_GAUGE_FIELD u_, 
                     INTEGER nn_fwd_[], 
                     INTEGER nn_bwd_[],
#ifdef BLOCK
                     int *i1, int *i2)
#else
                     int *volh)
#endif

{
    int  i, jf, jb;
    __m128d af_ri11,af_ri21,af_ri12,af_ri22,af_ri13,af_ri23;
    __m128d af_ir11,af_ir21,af_ir12,af_ir22,af_ir13,af_ir23;
    __m128d ab_ri11,ab_ri21,ab_ri12,ab_ri22,ab_ri13,ab_ri23;
    __m128d ab_ir11,ab_ir21,ab_ir12,ab_ir22,ab_ir13,ab_ir23;
    __m128d u_rr11,u_rr12,u_rr13,u_rr21,u_rr22,u_rr23,u_rr31,u_rr32,u_rr33;
    __m128d u_rr41,u_rr42,u_rr43,u_rr51,u_rr52,u_rr53,u_rr61,u_rr62,u_rr63;
    __m128d u_ii11,u_ii12,u_ii13,u_ii21,u_ii22,u_ii23,u_ii31,u_ii32,u_ii33;
    __m128d u_ii41,u_ii42,u_ii43,u_ii51,u_ii52,u_ii53,u_ii61,u_ii62,u_ii63;
    __m128d bf11,bf21,bf12,bf22,bf13,bf23;
    __m128d bb14,bb24,bb15,bb25,bb16,bb26;
    __m128d out11, out21, out31, out41, out12, out22, out32, out42, out13, out23, out33, out43;
    __m128d tmp11, tmp21, tmp31, tmp41, tmp12, tmp22, tmp32, tmp42, tmp13, tmp23, tmp33, tmp43;

    const __m128d conjg  = _mm_setr_pd(ONE, -ONE);
    const __m128d conjgr = _mm_set_pd(ONE, -ONE);


#ifndef PRECISION_R4
#ifndef BLOCK
    TIMING_START(STRCAT(timing_bin_, NAME));
#endif
#endif

    out_--;
    a_fwd_--;
    a_bwd_--;
    u_--;
    nn_fwd_--;
    nn_bwd_--;


#ifdef BLOCK
    for (i = *i1; i <= *i2; i++)
#else
    #pragma omp parallel for \
    private (i, jf, jb) \
    private (af_ri11,af_ri21,af_ri12,af_ri22,af_ri13,af_ri23) \
    private (af_ir11,af_ir21,af_ir12,af_ir22,af_ir13,af_ir23) \
    private (ab_ri11,ab_ri21,ab_ri12,ab_ri22,ab_ri13,ab_ri23) \
    private (ab_ir11,ab_ir21,ab_ir12,ab_ir22,ab_ir13,ab_ir23) \
    private (u_rr11,u_rr12,u_rr13,u_rr21,u_rr22,u_rr23,u_rr31,u_rr32,u_rr33) \
    private (u_rr41,u_rr42,u_rr43,u_rr51,u_rr52,u_rr53,u_rr61,u_rr62,u_rr63) \
    private (u_ii11,u_ii12,u_ii13,u_ii21,u_ii22,u_ii23,u_ii31,u_ii32,u_ii33) \
    private (u_ii41,u_ii42,u_ii43,u_ii51,u_ii52,u_ii53,u_ii61,u_ii62,u_ii63) \
    private (bf11,bf21,bf12,bf22,bf13,bf23) \
    private (bb14,bb24,bb15,bb25,bb16,bb26) \
    private (out11, out21, out31, out41, out12, out22, out32, out42, out13, out23, out33, out43) \
    private (tmp11, tmp21, tmp31, tmp41, tmp12, tmp22, tmp32, tmp42, tmp13, tmp23, tmp33, tmp43)

    for (i = 1; i <= *volh; i++)
#endif
    {
	jf=nn_fwd(i);
	jb=nn_bwd(i);

        cLoadAf(1, 1);
        cLoadAf(2, 1);
        cLoadU(1, 1);
        cLoadU(2, 1);
        cLoadU(3, 1);
        cProdAfU(1, 1, 1);
        cProdAfU(2, 1, 1);
        cProdAfU(1, 2, 1);
        cProdAfU(2, 2, 1);
        cProdAfU(1, 3, 1);
        cProdAfU(2, 3, 1);

        cLoadAb(1, 1);
        cLoadAb(2, 1);
        cLoadU(4, 1);
        cLoadU(5, 1);
        cLoadU(6, 1);
        cProdAbU(1, 4, 1);
        cProdAbU(2, 4, 1);
        cProdAbU(1, 5, 1);
        cProdAbU(2, 5, 1);
        cProdAbU(1, 6, 1);
        cProdAbU(2, 6, 1);

        cLoadAf(1, 2);
        cLoadAf(2, 2);
        cLoadU(1, 2);
        cLoadU(2, 2);
        cLoadU(3, 2);
        cAddProdAfU(1, 1, 2);
        cAddProdAfU(2, 1, 2);
        cAddProdAfU(1, 2, 2);
        cAddProdAfU(2, 2, 2);
        cAddProdAfU(1, 3, 2);
        cAddProdAfU(2, 3, 2);

        cLoadAb(1, 2);
        cLoadAb(2, 2);
        cLoadU(4, 2);
        cLoadU(5, 2);
        cLoadU(6, 2);
        cAddProdAbU(1, 4, 2);
        cAddProdAbU(2, 4, 2);
        cAddProdAbU(1, 5, 2);
        cAddProdAbU(2, 5, 2);
        cAddProdAbU(1, 6, 2);
        cAddProdAbU(2, 6, 2);
	
        cLoadAf(1, 3);
        cLoadAf(2, 3);
        cLoadU(1, 3);
        cLoadU(2, 3);
        cLoadU(3, 3);
        cAddProdAfU(1, 1, 3);
        cAddProdAfU(2, 1, 3);
        cAddProdAfU(1, 2, 3);
        cAddProdAfU(2, 2, 3);
        cAddProdAfU(1, 3, 3);
        cAddProdAfU(2, 3, 3);

        cLoadAb(1, 3);
        cLoadAb(2, 3);
        cLoadU(4, 3);
        cLoadU(5, 3);
        cLoadU(6, 3);
        cAddProdAbU(1, 4, 3);
        cAddProdAbU(2, 4, 3);
        cAddProdAbU(1, 5, 3);
        cAddProdAbU(2, 5, 3);
        cAddProdAbU(1, 6, 3);
        cAddProdAbU(2, 6, 3);


#ifdef DIR_T
	cLoadOut(1, 1);
        cLoadOut(2, 1);
        cLoadOut(3, 1);
        cLoadOut(4, 1);
        cLoadOut(1, 2);
        cLoadOut(2, 2);
        cLoadOut(3, 2);
        cLoadOut(4, 2);
        cLoadOut(1, 3);
        cLoadOut(2, 3);
        cLoadOut(3, 3);
        cLoadOut(4, 3);
#ifdef DAGGER
        cAdd(out11, bf11);
        cAdd(out21, bf21);
        cAdd(out31, bb14);
        cAdd(out41, bb24);
        cAdd(out12, bf12);
        cAdd(out22, bf22);
        cAdd(out32, bb15);
        cAdd(out42, bb25);
        cAdd(out13, bf13);
        cAdd(out23, bf23);
        cAdd(out33, bb16);
        cAdd(out43, bb26);
#else
        cAdd(out11, bb14);
        cAdd(out21, bb24);
        cAdd(out31, bf11);
        cAdd(out41, bf21);
        cAdd(out12, bb15);
        cAdd(out22, bb25);
        cAdd(out32, bf12);
        cAdd(out42, bf22);
        cAdd(out13, bb16);
        cAdd(out23, bb26);
        cAdd(out33, bf13);
        cAdd(out43, bf23);
#endif
	cStoreOut(1, 1);
        cStoreOut(2, 1);
        cStoreOut(3, 1);
        cStoreOut(4, 1);
        cStoreOut(1, 2);
        cStoreOut(2, 2);
        cStoreOut(3, 2);
        cStoreOut(4, 2);
        cStoreOut(1, 3);
        cStoreOut(2, 3);
        cStoreOut(3, 3);
        cStoreOut(4, 3);
#endif

#ifdef DIR_X
	cSum(out11, bf11, bb14);
	cSum(out21, bf21, bb24);
	cDiff3(out31, bf21, bb24);
	cDiff3(out41, bf11, bb14);

	cSum(out12, bf12, bb15);
	cSum(out22, bf22, bb25);
	cDiff3(out32, bf22, bb25);
	cDiff3(out42, bf12, bb15);

	cSum(out13, bf13, bb16);
	cSum(out23, bf23, bb26);
	cDiff3(out33, bf23, bb26);
	cDiff3(out43, bf13, bb16);

	cStoreOut(1, 1);
        cStoreOut(2, 1);
        cStorerOut(3, 1);
        cStorerOut(4, 1);
        cStoreOut(1, 2);
        cStoreOut(2, 2);
        cStorerOut(3, 2);
        cStorerOut(4, 2);
        cStoreOut(1, 3);
        cStoreOut(2, 3);
        cStorerOut(3, 3);
        cStorerOut(4, 3);
#endif


#ifdef DIR_Y
        cSum(tmp11, bf11, bb14);
        cSum(tmp21, bf21, bb24);
        cDiff2(tmp31, bf21, bb24);
        cDiff2(tmp41, bb14, bf11);

        cSum(tmp12, bf12, bb15);
        cSum(tmp22, bf22, bb25);
        cDiff2(tmp32, bf22, bb25);
        cDiff2(tmp42, bb15, bf12);

        cSum(tmp13, bf13, bb16);
        cSum(tmp23, bf23, bb26);
        cDiff2(tmp33, bf23, bb26);
        cDiff2(tmp43, bb16, bf13);

	cLoadOut(1, 1);
        cLoadOut(2, 1);
        cLoadOut(3, 1);
        cLoadOut(4, 1);
        cLoadOut(1, 2);
        cLoadOut(2, 2);
        cLoadOut(3, 2);
        cLoadOut(4, 2);
        cLoadOut(1, 3);
        cLoadOut(2, 3);
        cLoadOut(3, 3);
        cLoadOut(4, 3);

        cAdd(out11, tmp11);
        cAdd(out21, tmp21);
        cAdd(out31, tmp31);
        cAdd(out41, tmp41);
        cAdd(out12, tmp12);
        cAdd(out22, tmp22);
        cAdd(out32, tmp32);
        cAdd(out42, tmp42);
        cAdd(out13, tmp13);
        cAdd(out23, tmp23);
        cAdd(out33, tmp33);
        cAdd(out43, tmp43);

	cStoreOut(1, 1);
        cStoreOut(2, 1);
        cStoreOut(3, 1);
        cStoreOut(4, 1);
        cStoreOut(1, 2);
        cStoreOut(2, 2);
        cStoreOut(3, 2);
        cStoreOut(4, 2);
        cStoreOut(1, 3);
        cStoreOut(2, 3);
        cStoreOut(3, 3);
        cStoreOut(4, 3);
#endif
	
#ifdef DIR_Z
        cSum(tmp11, bf11, bb14);
        cSum(tmp21, bf21, bb24);
        cDiff3(tmp31, bf11, bb14);
        cDiff3(tmp41, bb24, bf21);

        cSum(tmp12, bf12, bb15);
        cSum(tmp22, bf22, bb25);
        cDiff3(tmp32, bf12, bb15);
        cDiff3(tmp42, bb25, bf22);

        cSum(tmp13, bf13, bb16);
        cSum(tmp23, bf23, bb26);
        cDiff3(tmp33, bf13, bb16);
        cDiff3(tmp43, bb26, bf23);

	cLoadOut(1, 1);
        cLoadOut(2, 1);
        cLoadrOut(3, 1);
        cLoadrOut(4, 1);
        cLoadOut(1, 2);
        cLoadOut(2, 2);
        cLoadrOut(3, 2);
        cLoadrOut(4, 2);
        cLoadOut(1, 3);
        cLoadOut(2, 3);
        cLoadrOut(3, 3);
        cLoadrOut(4, 3);

        cAdd(out11, tmp11);
        cAdd(out21, tmp21);
        cAdd(out31, tmp31);
        cAdd(out41, tmp41);
        cAdd(out12, tmp12);
        cAdd(out22, tmp22);
        cAdd(out32, tmp32);
        cAdd(out42, tmp42);
        cAdd(out13, tmp13);
        cAdd(out23, tmp23);
        cAdd(out33, tmp33);
        cAdd(out43, tmp43);

	cStoreOut(1, 1);
        cStoreOut(2, 1);
        cStorerOut(3, 1);
        cStorerOut(4, 1);
        cStoreOut(1, 2);
        cStoreOut(2, 2);
        cStorerOut(3, 2);
        cStorerOut(4, 2);
        cStoreOut(1, 3);
        cStoreOut(2, 3);
        cStorerOut(3, 3);
        cStorerOut(4, 3);
#endif
    }
    
#ifndef PRECISION_R4
#ifndef BLOCK
    TIMING_STOP(STRCAT(timing_bin_, NAME));
#endif
#endif

}

# undef cCopy
# undef cCopyr
# undef cSum
# undef cDiff
# undef cDiff2
# undef cDiff2
# undef cDiff3
# undef cProd
# undef cAdd
# undef cAddSum
# undef cAddProd
# undef  cLoadAf
# undef  cLoadAb
# undef   cLoadU
# undef cProdAfU
# undef cProdAbU
# undef cAddProdAfU
# undef cAddProdAbU
# undef cLoadOut
# undef cLoadrOut
# undef cStoreOut
# undef cStorerOut
