//==============================================================================
//
// d/wilson_d500/cr_asm.cc
//
//------------------------------------------------------------------------------
//
// Copyright (C) 2006 Thomas Streuer
//
// This file is part of BQCD -- Berlin Quantum ChromoDynamics program
//
// BQCD is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// BQCD is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
//
//------------------------------------------------------------------------------
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
 
#ifndef SINGLE
const int rm_send_mask=0x4000;
const int rm_recv_mask=0x8000;
#endif

#define DEBUG_PREF_
#define DEBUG_PREF_U_FW_
#define DEBUG_PREF_U_BW_
#define DEBUG_PREF_PSI_L_
#define DEBUG_PREF_PSI_R_

#define NO_SYNC_

const int max_diff                    =  1;


const int wd_offs_neighb_offs         =  0;
const int wd_offs_neighb_offs_even    =  4;
const int wd_offs_neighb_offs_odd     =  8;
const int wd_offs_neighb_offs_rm      = 12;
const int wd_offs_neighb_offs_rm_even = 16;
const int wd_offs_neighb_offs_rm_odd  = 20;
const int wd_offs_volume              = 24;
const int wd_offs_volh_tot            = 28;

const int wd_offs_gpr = 32;
const int wd_offs_fpr = 160;
const int wd_offs_spr = 672;
const int wd_offs_fs = 688;
const int wd_offs_neighb_offs_l       = 752;
const int wd_offs_neighb_offs_l_even  = 756;
const int wd_offs_neighb_offs_l_odd   = 760;

const int fifo4_mask   = 0x3ff;
const int fifo4_size   =  1024;   // size of the memory fifos in bytes
const int fifo_size    =   32;   // size of the torus fifos in 32b-Blocks
bool jupp;
int phys_dir[4];

int stack_size;
int n_gpr=32;
int n_fr=32;
bool mainloop=false;
bool dagger=false;

int reg_0             =  0;
int reg_stack         =  1;
int reg_toc           =  2;
int reg_load_U_fw     =  3;         // fixed (input)
int reg_Pinx          =  4;         // fixed (input)
int reg_Pout          =  5;         // fixed (input)
int reg_oe            =  6;         // fixed (input)


int reg_nb            = -1;
int reg_nb_rm         = -1;
int reg_nb_l          = -1;
int reg_nb_pref_r     = -1;
int reg_16            = -1;
int reg_48            = -1;
int reg_144           = -1;
int reg_m80           = -1;
int reg_80            = -1;   // !!!!!
int reg_m176          = -1;   // !!!!!
int reg_load_psi      = -1;
int reg_load_psi_r    = -1;
int reg_pref_psi_l    = -1;
int reg_pref_psi_r    = -1;
int reg_load_U_bw     = -1;
int reg_fifo_stat     = -1;
int reg_Ufw_offs      = -1;
int reg_pref_U_fw     = -1;
int reg_pref_U_bw     = -1;
int reg_data          = -1;


#ifndef SINGLE

const int bit_nb         = 24;
const int bit_pr_rm      = 16;
const int bit_send_ok    = 8;
const int bit_recv_ok    = 7;

const int send_stat_byte_offs[2][3]={{7, 8, 9}, {7, 8, 9}};
const int recv_stat_byte_offs[2][3]={{0, 2, 4}, {1, 3, 5}};

#endif

const int pr_ahead       = 1;
const int pref_dist      = 2;

int st_off_jupp;
int st_off_save_regs;
int st_off_const_i;
int st_off_kappa;
int st_off_scratch16;
#ifdef DEBUG_PREF
int st_off_debug_pref;
#endif

#ifndef SINGLE
int st_off_recv_stat_addr;
int st_off_send_stat_addr;
int st_off_recv_fifo_addr;
int st_off_send_fifo_addr;
int st_off_send_fifo4_addr;
int st_off_recv_fifo4_addr;
int st_off_send_header;

const int jupp_fifo_size=24576; // size of the fake jupp fifos

#endif



int fpreg_U[9];
int fpreg_psi_pr[6];
int fpreg_hres[6];
int fpreg_hres_orig[6];
int fpreg_res[12];
int fpreg_kappa1;

int fpreg_rem_U[9];
int fpreg_rem_psi_pr[6];


class save_regs
{
public: 
  save_regs();
  void restore_regs();
private:
  int reg_U[3];
};

save_regs::save_regs()
{
  for (int i=0; i<3; i++)
    reg_U[i]=fpreg_U[i];
}

void save_regs::restore_regs()
{
  for (int i=0; i<3; i++)
    fpreg_U[i]=reg_U[i];
}

class asm_stack
{
public:
  asm_stack();
  int add(int sz);
  int get_size() {return size;}
private:
  int size;
};

asm_stack::asm_stack()
{
  size=0;
}

int asm_stack::add(int sz)
{
  size+=sz;
  return size-sz;
}

class asm_code_element
{
public:
  char* str;
  bool jump;
  bool fixed;
  int n_res_mod, n_res_used, type;
  int* res_mod,* res_used;
  asm_code_element();
  asm_code_element(char* _str);
  asm_code_element(char* _str, int n_mod, int* mod,
                   int n_used, int* used, bool _fixed, bool _jump);
  void print();
};
const int asm_code_max_el=50000;

class asm_code
{
  asm_code_element** el;
  int n_el;
public:
  asm_code();
  void init();
  void add(asm_code_element* e);
  void optimize();
  void listing();
};


asm_code_element::asm_code_element()
{
  str=0;
}

asm_code_element::asm_code_element(char* _str)
{
  str=new char[strlen(_str)+1];
  strcpy(str, _str);
}  

asm_code_element::asm_code_element(
  char* _str, int n_mod, int* mod, int n_used, int* used,
  bool _fixed, bool _jump)
{
  str=new char[strlen(_str)+1];
  strcpy(str, _str);
  n_res_mod=n_mod;
  n_res_used=n_used;
  res_mod=new int[n_mod];
  res_used=new int[n_used];
  for (int i=0; i<n_mod; i++)
    res_mod[i]=mod[i];
  for (int i=0; i<n_used; i++)
    res_used[i]=used[i];
  fixed=_fixed;
  jump=_jump;
}


void asm_code_element::print()
{
  printf("%s\n", str);
}

asm_code::asm_code()
{
  el=new asm_code_element*[asm_code_max_el];
  n_el=0;
}

void asm_code::init()
{
  if (el!=0)
    delete [] el;
  el=new asm_code_element*[asm_code_max_el];
  n_el=0;
}

void asm_code::add(asm_code_element* e)
{
  el[n_el++]=e;
}

void asm_code::listing()
{
  for (int i=0; i<n_el; i++)
    el[i]->print();
}

// 0 : 1, 1 : i, 2 : -i, 3 : -1
int hsp_isign[3][2] = {{1, 1}, {0, 3}, {1, 2}};
int hsp_col[3][2]   = {{1, 0}, {1, 0}, {0, 1}};



class registers
{
private:
  int n_regs, max_reg;
  int* reg_state;
public:
  static const int reg_free    = -1;
  static const int reg_U       =  1;
  static const int reg_psi_pr  =  2;
  static const int reg_hres    =  3;
  static const int reg_coll    =  4;
  static const int reg_scratch =  5;
  static const int reg_res     =  6;
  registers(int n);
  void reset();
  void get_regs(int num, int purpose, int* r);
  void get_these_regs(int num, int purpose, int* r);
  void release_regs(int num, int* r);
  int get_max_reg() {return max_reg;}
};

registers::registers(int n)
{
  n_regs=n;
  reg_state=new int[n_regs];
  for (int i=0; i<n_regs; i++)
    reg_state[i]=reg_free;
  max_reg=0;
}

void registers::reset()
{
  for (int i=0; i<n_regs; i++)
    reg_state[i]=reg_free;
}

void registers::get_regs(int num, int purpose, int* r)
{
  int i=0;
  while (i<n_regs && num>0)
  {
    if (reg_state[i]==reg_free)
    {
      reg_state[i]=purpose;
      *r++=i;
      num--;
      if (i>max_reg)
	max_reg=i;
    }
    i++;
  }
  if (num>0)
  {
    printf("error allocating registers (%i, %i)\n", num, purpose);
    for (int i=0; i<n_regs; i++)
      printf("%2i\t% 3i\n", i, reg_state[i]);
    exit(1);
  }
}

void registers::get_these_regs(int num, int purpose, int* r)
{
  for (int i=0; i<num; i++)
    if (reg_state[r[i]]==reg_free)
    {
      reg_state[r[i]]=purpose;
      if (r[i]>max_reg)
	max_reg=r[i];
    }
    else
    {
      printf("error allocating these registers\n");
      for (int i=0; i<n_regs; i++)
	printf("%2i\t% 3i\n", i, reg_state[i]);
      exit(1);
    }
}

void registers::release_regs(int num, int* r)
{
  for (int i=0; i<num; i++)
    reg_state[*r++]=reg_free;
}


asm_code code;
asm_stack stack;
registers fpregs(32);
registers gpregs(32);

int cr_bit(int base, int f, int mu)
{
  return base+4*f+mu;
}

char* get_label()
{
  static int label_count=0;
  char* s=new char[20];
  sprintf(s, "dirac_label_%04i", label_count++);
  return s;
}

char* add_label(char* l=0)
{
  if (l==0)
    l=get_label();
  char str[200];
  sprintf(str, "%s:", l);
  code.add(new asm_code_element(str));
  return l;
}

void jump_c(char* label, int cond=-1, bool cond_true=true)
{
  char str[200];
  if (cond==-1)
    sprintf(str, "b  %s", label);
  else
    sprintf(str, "bc  %i, %i, %s", cond_true? 12 : 4, cond, label);
  code.add(new asm_code_element(str));
}

void upd_lr()
{
  char* label=get_label();
  char str[200];
  sprintf(str, "bl  %s", label);
  code.add(new asm_code_element(str));
  add_label(label);
}  
  
void comment(char* c)
{
  char str[80];
  sprintf(str, "#%s\n", c);
  code.add(new asm_code_element(str));
}



void load(int target, int offs, int reg, bool upd=false)
{
  char str[255];
  sprintf(str, "      lwz%s %i, %i(%i)", 
	  upd? "u" : "", target, offs, reg);
  code.add(new asm_code_element(str));
}

void load_hword_a(int target, int offs, int reg, bool upd=false)
{
  char str[255];
  sprintf(str, "      lha%s %i, %i(%i)", 
	  upd? "u" : "", target, offs, reg);
  code.add(new asm_code_element(str));
}

void load_byte(int target, int offs, int reg)
{
  char str[255];
  sprintf(str, "      lbz %i, %i(%i)", target, offs, reg);
  code.add(new asm_code_element(str));
}

void load(int target, char* offs, int reg)
{
  char str[255];
  sprintf(str, "      lwz %i, %s(%i)", target, offs, reg);
  code.add(new asm_code_element(str));
}

void load_hi(int target, char* expr)
{
  char str[255];
  sprintf(str, "      addis %i, 0, %s@ha", target, expr);
  code.add(new asm_code_element(str));
}

void load_lo(int target, char* expr)
{
  char str[255];
  sprintf(str, "      addi %i, %i, %s@l", target, target, expr);
  code.add(new asm_code_element(str));
}


void load_i(int target, int imm)
{
  char str[255];
  sprintf(str, "      li %i, %i", target, imm);
  code.add(new asm_code_element(str));
}

void load_i32(int target, int imm)
{
  char str[255];
  if (imm & 0x8000)
    sprintf(str, "      li %i, %i", target, (imm & 0xffff)-0x10000);
  else
    sprintf(str, "      li %i, %i", target, imm & 0xffff);
  code.add(new asm_code_element(str));
  if (imm & 0x8000)
    sprintf(str, "      addis %i, %i, %i", target, target, (imm >> 16)+1);
  else
    sprintf(str, "      addis %i, %i, %i", target, target, imm >> 16);
  code.add(new asm_code_element(str));
}

void store(int source, int offs, int reg)
{
  char str[255];
  sprintf(str, "      stw %i, %i(%i)", source, offs, reg);
  code.add(new asm_code_element(str));
}



void add(int target, int src1, int src2)
{
  char str[255];
  sprintf(str, "      add %i, %i, %i", target, src1, src2);
  code.add(new asm_code_element(str));
}

void add_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      addi %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void add_is(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      addis %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void and_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      andi. %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void and_is(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      andis. %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void mul_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      mulli %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void shiftl_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      rlwinm %i, %i, %i, 0, %i", target, src, imm, 31-imm);
  code.add(new asm_code_element(str));
}

void shiftr_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      srawi %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void mov_cr(int dest, int src, bool neg=false)
{
  char str[255];
  sprintf(str, "      cr%sand %i, %i, %i",
   	  neg? "n" : "", dest, src, src);
  code.add(new asm_code_element(str));
}

void set_cr(int dest, bool one=true)
{
  char str[255];
  sprintf(str, "      cr%s %i, %i, %i",
   	  one? "eqv" : "xor", dest, dest, dest);
  code.add(new asm_code_element(str));
}

void sub(int target, int src1, int src2, bool upd=false)
{
  char str[255];
  sprintf(str, "      subfc%s %i, %i, %i",
	  upd? "." : "", target, src2, src1);
  code.add(new asm_code_element(str));
}

void cmp(int target, int src1, int src2)
{
  char str[255];
  sprintf(str, "      cmp %i, %i, %i", target, src1, src2);
  code.add(new asm_code_element(str));
}

void cmp_i(int target, int src, int imm)
{
  char str[255];
  sprintf(str, "      cmpi %i, %i, %i", target, src, imm);
  code.add(new asm_code_element(str));
}

void mac(int target, int src1, int src2)
{
  char str[255];
  sprintf(str, "      maclhw %i, %i, %i", target, src1, src2);
  code.add(new asm_code_element(str));
}


void fload(int target, int reg1, int reg2, bool par=false, bool upd=false, bool cross=false)
{
  char str[255];
  sprintf(str, "      lf%sd%sx %i, %i, %i", par? (cross? "x" : "p") : "", upd? "u" : "", target, reg1, reg2);
  code.add(new asm_code_element(str));
}

void fload8(int target, int offs, int reg)
{
  char str[255];
  sprintf(str, "      lfd %i, %i(%i)", target, offs, reg);
  code.add(new asm_code_element(str));
}

void fload8x(int target, int reg1, int reg2, bool sec=false)
{
  char str[255];
  sprintf(str, "      lf%sdx %i, %i, %i", sec? "s" : "",
	  target, reg1, reg2);
  code.add(new asm_code_element(str));
}

void fstore(int src, int reg1, int reg2, bool par=false, bool upd=false)
{
  char str[255];
  sprintf(str, "      stf%sd%sx %i, %i, %i", par? "p" : "", upd? "u" : "", src, reg1, reg2);
  code.add(new asm_code_element(str));
}

void fstore8x(int src, int reg1, int reg2, bool sec=false)
{
  char str[255];
  sprintf(str, "      stf%sdx %i, %i, %i", sec? "s" : "", src, reg1, reg2);
  code.add(new asm_code_element(str));
}

void fstore_offs(int src, int offs, int reg)
{
  char str[255];
  sprintf(str, "      stfd  %i, %i(%i)", src, offs, reg);
  code.add(new asm_code_element(str));
}

void fload_offs(int dest, int offs, int reg)
{
  char str[255];
  sprintf(str, "      lfd  %i, %i(%i)", dest, offs, reg);
  code.add(new asm_code_element(str));
}

void prefetch(int reg1, int reg2)
{
  char str[255];
  sprintf(str, "      dcbt %i, %i", reg1, reg2);
  if (!jupp)
    code.add(new asm_code_element(str));
}


void fmul(int dest, int src1, int src2)
{
  char str[255];
  sprintf(str, "fmul   %i, %i, %i", dest, src1, src2);
  code.add(new asm_code_element(str));
}

// multiply src2 by primary(src1) and add to src3
void fmuladd_s(int dest, int src1, int src2, int src3)
{
  char str[255];
  sprintf(str, "fxcpmadd   %i, %i, %i, %i", 
	    dest, src1, src2, src3);
  code.add(new asm_code_element(str));
}


void fadd(int dest, int a, int b, bool par=false)
{
  char str[255];
  sprintf(str, "f%sadd   %i, %i, %i", 
	  par? "p" : "", dest, a, b);
  code.add(new asm_code_element(str));
}

void fsub(int dest, int a, int b, bool par=false)
{
  char str[255];
  sprintf(str, "f%ssub   %i, %i, %i", 
	  par? "p" : "", dest, a, b);
  code.add(new asm_code_element(str));
}

// i    dest

// 0    dest + src
// 1    dest + i * src
// 2    dest - i * src
// 3    dest - src

// freg_i must contain "1"in secondary reg.

void fadd_isign(int dest, int src, int isign, int freg_i, int acc=-1)
{
  if (acc==-1)
    acc=dest;
  char str[255];
  if (isign==0 || isign==3)
    sprintf(str, "fp%s   %i, %i, %i", 
	  isign==0? "add" : "sub", dest, acc, src);
  else if (isign==1 || isign==2)
    sprintf(str, "fxcxn%sma   %i, %i, %i, %i", 
	  isign==1? "p" : "s", dest, freg_i, src, acc);
  
  code.add(new asm_code_element(str));
}


// dest=dest+/- a*b
// pass: 0/1
// dg: if true, conjugate a
void cplxmul(int dest, int a, int b, int pass, int add=true, bool cj=false, bool negate=false, int acc=-1)
{
  char str[255];
  if (acc==-1)
    acc=dest;
  if (negate)
  {
    if (add)
      sprintf(str, "%s   %i, %i, %i, %i", 
	      pass==0? "fxcpnmadd" : (cj? "fxcxnpma" : "fxcxnsma"),
	      dest, a, b, acc);
    else if (pass==0)
      sprintf(str, "fxpmul   %i, %i, %i", 
	      dest, a, b);
  }
  else
    if (add)
      sprintf(str, "%s   %i, %i, %i, %i", 
	      pass==0? "fxcpmadd" : (cj? "fxcxnsma" : "fxcxnpma"),
	      dest, a, b, acc);
    else if (pass==0)
      sprintf(str, "fxpmul   %i, %i, %i", 
	      dest, a, b);
    else
  {
    printf("error in cplxmul\n");
    exit(1);
  }
  code.add(new asm_code_element(str));
}



void fmov(int dest, int src, bool par=false)
{
  char str[255];
  sprintf(str, "f%smr   %i, %i", par? "p" : "", dest, src);
  code.add(new asm_code_element(str));
}

void fneg(int dest, int src, bool sec=false, bool par=false)
{
  char str[255];
  if (par)
    sprintf(str, "fpneg   %i, %i", dest, src);
  else
    sprintf(str, "f%sneg   %i, %i", sec? "s" : "", dest, src);
  code.add(new asm_code_element(str));
}

#ifndef SINGLE

// check whether send fifo is ready for halfspinor
// result: cr 0=1 ==> ok
void send_fifo_check(int f, int mu, int freg_scratch)
{
  if (jupp)
  {
    char str[200];
    store(20, st_off_jupp, 1);
    store(21, st_off_jupp, 1);
    store(22, st_off_jupp, 1);
    store(23, st_off_jupp, 1);
    code.add(new asm_code_element("sync"));
    load(20, "addr_send_fifo", 2);
    code.add(new asm_code_element("sync"));
    load(20, 4*(4*f+mu), 20);
    code.add(new asm_code_element("sync"));
    load(21, 4, 20); // wr ptr
    code.add(new asm_code_element("sync"));
    load(22, 0, 20); // rd ptr
    code.add(new asm_code_element("sync"));
    cmp(0, 21, 22);
    char* jmp_label=get_label();
    sprintf(str, "bc 4, 0, %s", jmp_label);
    code.add(new asm_code_element(str));
    add_i(21, 21, jupp_fifo_size);
    sprintf(str, "%s:", jmp_label);
    code.add(new asm_code_element(str));
    delete [] jmp_label;
    add_i(21, 21, -jupp_fifo_size+200);
    cmp(0, 21, 22);
    load(20, st_off_jupp, 1);
    load(21, st_off_jupp, 1);
    load(22, st_off_jupp, 1);
    load(23, st_off_jupp, 1);
  }
  else
    if (phys_dir[mu]!=3)
    {
      int reg_scratch[2];
      gpregs.get_regs(2, 0, reg_scratch);
      load_i(reg_scratch[1], st_off_scratch16);
      fload(freg_scratch, 0, reg_fifo_stat, true);
      fstore(freg_scratch, reg_scratch[1], 1, true);
      load_byte(reg_scratch[0], st_off_scratch16+send_stat_byte_offs[f][phys_dir[mu]], 1);
      cmp_i(0, reg_scratch[0], 24);
      gpregs.release_regs(2, reg_scratch);
    }
    else
    {
      int reg_scratch[2];
      gpregs.get_regs(2, 0, reg_scratch);
      load(reg_scratch[0], st_off_send_fifo4_addr, 1);
      load(reg_scratch[1], 0, reg_scratch[0]); // rd offs
      load(reg_scratch[0], 4, reg_scratch[0]);   // wr offs
      sub(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
      cmp_i(0, reg_scratch[0], fifo4_size-128);
      gpregs.release_regs(2, reg_scratch);
    }
}

// check whether recv fifo contains data (halfspinor)
// result: cr 1=1 ==> ok
void recv_fifo_check(int f, int mu, int freg_scratch)
{
  if (jupp)
  {
    char str[200];
    store(20, st_off_jupp, 1);
    store(21, st_off_jupp, 1);
    store(22, st_off_jupp, 1);
    store(23, st_off_jupp, 1);
    code.add(new asm_code_element("sync"));
    load(20, "addr_recv_fifo", 2);
    code.add(new asm_code_element("sync"));
    load(20, 4*(4*f+mu), 20);
    code.add(new asm_code_element("sync"));
    load(21, 4, 20); // wr ptr
    code.add(new asm_code_element("sync"));
    load(22, 0, 20); // rd ptr
    code.add(new asm_code_element("sync"));
    cmp(0, 21, 22);
    char* jmp_label=get_label();
    sprintf(str, "bc 4, 0, %s", jmp_label);
    code.add(new asm_code_element(str));
    add_i(21, 21, jupp_fifo_size);
    sprintf(str, "%s:", jmp_label);
    code.add(new asm_code_element(str));
    delete [] jmp_label;
    add_i(21, 21, -95);
    cmp(0, 21, 22);
    load(20, st_off_jupp, 1);
    load(21, st_off_jupp, 1);
    load(22, st_off_jupp, 1);
    load(23, st_off_jupp, 1);
  }
  else
    if (phys_dir[mu]!=3)
    {
      int reg_scratch[2];
      gpregs.get_regs(2, 0, reg_scratch);
      load_i(reg_scratch[1], st_off_scratch16);
      fload(freg_scratch, 0, reg_fifo_stat, true);
      fstore(freg_scratch, reg_scratch[1], 1, true);
      load_byte(reg_scratch[0], st_off_scratch16+recv_stat_byte_offs[f][phys_dir[mu]], 1);
      cmp_i(0, reg_scratch[0], 0);
      gpregs.release_regs(2, reg_scratch);
    }
    else
    {
      int reg_scratch[2];
      gpregs.get_regs(2, 0, reg_scratch);
      load(reg_scratch[0], st_off_recv_fifo4_addr, 1);
      load(reg_scratch[1], 0, reg_scratch[0]); // rd offs
      load(reg_scratch[0], 4, reg_scratch[0]);   // wr offs
      sub(reg_scratch[0], reg_scratch[0], reg_scratch[1], true);
      gpregs.release_regs(2, reg_scratch);
    }
}

void fifo_send(int f, int mu, int reg, int s_reg=-1)
{
  if (jupp)
  {
    char str[200];
    store(20, st_off_jupp, 1);
    store(21, st_off_jupp, 1);
    store(22, st_off_jupp, 1);
    store(23, st_off_jupp, 1);
    code.add(new asm_code_element("sync"));
    load(20, "addr_send_fifo", 2);
    code.add(new asm_code_element("sync"));
    load(20, 4*(4*f+mu), 20);
    code.add(new asm_code_element("sync"));
    load(21, 4, 20); // wr ptr
    code.add(new asm_code_element("sync"));
    add_i(21, 21, 8);
    add(21, 21, 20);
    code.add(new asm_code_element("sync"));
    fstore(reg, 0, 21, true);
    code.add(new asm_code_element("sync"));
    add_i(21, 21, 8);
    sub(21, 21, 20);
    cmp_i(0, 21, jupp_fifo_size);
    char* jmp_label=get_label();
    sprintf(str, "bc 4, 2, %s", jmp_label);
    code.add(new asm_code_element(str));
    add_i(21, 21, -jupp_fifo_size);
    sprintf(str, "%s:", jmp_label);
    code.add(new asm_code_element(str));
    code.add(new asm_code_element("sync"));
    store(21, 4, 20);
    code.add(new asm_code_element("sync"));
    load(20, st_off_jupp, 1);
    load(21, st_off_jupp, 1);
    load(22, st_off_jupp, 1);
    load(23, st_off_jupp, 1);
  }
  else
    if (phys_dir[mu]!=3)
      fstore(reg, 0, s_reg, true);
    else
      fstore(reg, s_reg, reg_16, true, true);
}

void get_fifo_recv_addr(int f, int mu, int reg, int reg_offs, int reg_addr)
{
  if (jupp)
    return;
  if (phys_dir[mu]!=3)
    load(reg, st_off_recv_fifo_addr+4*(phys_dir[mu]+3*f), 1);
  else
  {
    load(reg_addr, st_off_recv_fifo4_addr, 1);
    load(reg_offs, 0, reg_addr);
    and_i(reg, reg_offs, fifo4_mask);
    add(reg, reg, reg_addr);
    add_i(reg_offs, reg_offs, 128);
  }
}

void put_fifo4_recv_addr(int reg_offs, int reg_addr)
{
  if (jupp)
    return;
  code.add(new asm_code_element("mbar"));
  store(reg_offs, 0, reg_addr);
}

void get_fifo_send_addr(int f, int mu, int reg, int reg_offs, int reg_addr)
{
  if (jupp)
    return;
  if (phys_dir[mu]!=3)
    load(reg, st_off_send_fifo_addr+4*(phys_dir[mu]+3*f), 1);
  else
  {
    load(reg_addr, st_off_send_fifo4_addr, 1);
    load(reg_offs, 4, reg_addr);
    and_i(reg, reg_offs, fifo4_mask);
    add(reg, reg, reg_addr);
    add_i(reg_offs, reg_offs, 128);
  }
}

void put_fifo4_send_addr(int reg_offs, int reg_addr)
{
  if (jupp)
    return;
  code.add(new asm_code_element("mbar"));
  store(reg_offs, 4, reg_addr);
}

void fifo_recv(int f, int mu, int reg, int l_reg=-1)
{
  if (jupp)
  {
    char str[200];
    store(20, st_off_jupp, 1);
    store(21, st_off_jupp, 1);
    store(22, st_off_jupp, 1);
    store(23, st_off_jupp, 1);
    code.add(new asm_code_element("sync"));
    load(20, "addr_recv_fifo", 2);
    code.add(new asm_code_element("sync"));
    load(20, 4*(4*f+mu), 20);
    code.add(new asm_code_element("sync"));
    load(21, 0, 20); // rd ptr
    code.add(new asm_code_element("sync"));
    add_i(21, 21, 8);
    add(21, 21, 20);
    code.add(new asm_code_element("sync"));
    fload(reg, 0, 21, true);
    code.add(new asm_code_element("sync"));
    add_i(21, 21, 8);
    sub(21, 21, 20);
    cmp_i(0, 21, jupp_fifo_size);
    char* jmp_label=get_label();
    sprintf(str, "bc 4, 2, %s", jmp_label);
    code.add(new asm_code_element(str));
    add_i(21, 21, -jupp_fifo_size);
    sprintf(str, "%s:", jmp_label);
    code.add(new asm_code_element(str));
    code.add(new asm_code_element("sync"));
    store(21, 0, 20);
    code.add(new asm_code_element("sync"));
    load(20, st_off_jupp, 1);
    load(21, st_off_jupp, 1);
    load(22, st_off_jupp, 1);
    load(23, st_off_jupp, 1);
  }
  else
    if (phys_dir[mu]!=3)
      fload(reg, 0, l_reg, true);
    else
      fload(reg, l_reg, reg_16, true, true);
}

#endif

int P2o_table[4][2]={{3,2},{3,2},{2,3},{0,0}};
int P2r_table[4]={1,0,1,0};                      // 0 : real, 1 : imag 
int P2sr_table[4][2]={{1,1},{0,1},{1,0},{0,0}};
int P2si_table[4][2]={{0,0},{0,1},{0,1},{0,0}};  // 0 : +   , 1 : -

int P1_off(int col, int hs, int dir, int f)
{
  if (dir!=3) 
    return 16*(4*col+hs);
  else
    return 16*(4*col+(hs+2*(1-f)));
}

bool P_pr_real(int mu)
{
  if (mu==1)
    return true;
  else return false;
}

// must be called 12 times (i=0..11) even for mu==3

void load_psi(int f, int mu, int i, int scratch=6)
{
  int target_reg=(i%2)*scratch+(i/2);
  int col=i/4;
  int hsp;
  int pass=(i%4)/2;
  hsp=i%2;

  if (pass==0)
    target_reg=2*col+hsp;
  else
    target_reg=scratch+hsp_col[mu][hsp];
  int reg_offs=reg_16;
  if (pass==1 && mu==3 && !dagger)
  {
    if (col==2 && hsp==1 && f==1)
      add_i(reg_load_psi, reg_load_psi, 32);
    return;
  }
  if (pass==1 && mu==3 && dagger)
  {
    if (col==2 && hsp==1 && f==0)
      add_i(reg_load_psi, reg_load_psi, 32);
    return;
  }
  if (mu==3 && hsp==0 && (col>0 || (!dagger && f==0) || (dagger && f==1)))
    reg_offs=reg_48;
  fload(fpreg_psi_pr[target_reg], reg_load_psi, reg_offs, true, true);

#ifdef DEBUG_PREF_PSI_L
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
  load(reg_scratch[0], st_off_debug_pref+4, 1);
  store(reg_load_psi, 0, reg_scratch[0]);
  add_i(reg_scratch[0], reg_scratch[0], 4);
  store(reg_scratch[0], st_off_debug_pref+4, 1);
  gpregs.release_regs(1, reg_scratch);
#endif

}

// i=0..5
void prefetch_psi_l(int i)
{
  int reg_offs;
  if (i%3==0)
    reg_offs=0;
  if (i%3==1)
    reg_offs=reg_16;
  if (i%3==2)
    reg_offs=reg_48;
  prefetch(reg_offs, reg_pref_psi_l);
#ifdef DEBUG_PREF_PSI_L
  int reg_scratch[2];
  gpregs.get_regs(2, 0, reg_scratch);
  load(reg_scratch[0], st_off_debug_pref, 1);
  add_i(reg_scratch[1], reg_offs, 0);
  add(reg_scratch[1], reg_scratch[1], reg_pref_psi_l);
  store(reg_scratch[1], 0, reg_scratch[0]);
  add_i(reg_scratch[0], reg_scratch[0], 4);
  store(reg_scratch[0], st_off_debug_pref, 1);
  gpregs.release_regs(2, reg_scratch);
#endif
  if (i%3==2)
    add_i(reg_pref_psi_l, reg_pref_psi_l, 96);
}

// i=0..5
void prefetch_psi_r(int i)
{
  int reg_offs;
  if (i%3==0)
    reg_offs=0;
  if (i%3==1)
    reg_offs=reg_16;
  if (i%3==2)
    reg_offs=reg_48;
 prefetch(reg_offs, reg_pref_psi_r);
#ifdef DEBUG_PREF_PSI_R
  int reg_scratch[2];
  gpregs.get_regs(2, 0, reg_scratch);
  load(reg_scratch[0], st_off_debug_pref, 1);
  add_i(reg_scratch[1], reg_offs, 0);
  add(reg_scratch[1], reg_scratch[1], reg_pref_psi_r);
  store(reg_scratch[1], 0, reg_scratch[0]);
  add_i(reg_scratch[0], reg_scratch[0], 4);
  store(reg_scratch[0], st_off_debug_pref, 1);
  gpregs.release_regs(2, reg_scratch);
#endif
  if (i%3==2)
    add_i(reg_pref_psi_r, reg_pref_psi_r, 96);
}



void psi_pr(int f, int mu, int i, int scratch=6)
{
  if (mu==3)
    fadd(fpreg_psi_pr[i], fpreg_psi_pr[i], fpreg_psi_pr[i], true);
  else
  {
    if (dagger)
      f=1-f;
    if ((f==0 && (hsp_isign[mu][hsp_col[mu][i%2]]%3==0))  ||
	(f==1 && (hsp_isign[mu][hsp_col[mu][i%2]]%3!=0)))
      fadd_isign(fpreg_psi_pr[i], fpreg_psi_pr[scratch+i%2], 
		 hsp_isign[mu][hsp_col[mu][i%2]],
		 fpreg_kappa1);
    else
      fadd_isign(fpreg_psi_pr[i], fpreg_psi_pr[scratch+i%2], 
		 3-hsp_isign[mu][hsp_col[mu][i%2]],
		 fpreg_kappa1);
  }
}

void load_psi_rem_data(int f, int mu, int i, int& col, int& pass, int& hsp)
{
  col=i/4;
  pass=i%2;
  if (pass==0)
    hsp=(i%4)/2;
  else
    hsp=hsp_col[mu][(i%4)/2];
}

// load the spinor which will be sent to the neighbour
// must be called 12 times

void load_psi_rem(int f, int mu, int i, int reg)
{
  int col, pass, hsp;
  int col_n, pass_n, hsp_n;
  int i_n=(i+1)%12;
  if (mu==3)
    i_n=(i+1)%6;
  load_psi_rem_data(f, mu, i, col, pass, hsp); 
  load_psi_rem_data(f, mu, i_n, col_n, pass_n, hsp_n); 
  if (mu==3 && pass==1)
    return;
  int my_pos=16*(hsp+2*(pass+2*col));
  int next_pos=16*(hsp_n+2*(pass_n+2*col_n));
  if (i_n==0)
    next_pos+=192;
  int offs=next_pos-my_pos;
  if (mu==3)
    if (hsp==0)
      offs=16;
    else
      if (f>=0 && col==2)
	offs=16;
      else
	offs=48;
  if (dagger && mu==3 && i==0 && f==1)
    add_i(reg_load_psi_r, reg_load_psi_r, 32);
  if (!dagger && mu==3 && i==0 && f==0)
    add_i(reg_load_psi_r, reg_load_psi_r, 32);

  fload(reg, 0, reg_load_psi_r, true, false);

#ifdef DEBUG_PREF_PSI_R
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
  load(reg_scratch[0], st_off_debug_pref+4, 1);
  store(reg_load_psi_r, 0, reg_scratch[0]);
  add_i(reg_scratch[0], reg_scratch[0], 4);
  store(reg_scratch[0], st_off_debug_pref+4, 1);
  gpregs.release_regs(1, reg_scratch);
#endif


  if (dagger && mu==3 && f==0 && i==10)
    offs+=32;
  if (!dagger && mu==3 && f==1 && i==10)
    offs+=32;
  add_i(reg_load_psi_r, reg_load_psi_r, offs);
}



// must be called 6 times
// reg: 2 fp registers
void psi_pr_rem(int f, int mu, int i, int* reg, int* reg2=0)
{
  int r[2];
  r[0]=reg[0];
  if (reg2==0)
    r[1]=reg[1];
  else
    r[1]=reg2[0];
  if (mu==3)
    fadd(reg[0], reg[0], reg[0], true);
  else
  {
    if (dagger)
      f=1-f;
    if ((f==0 && (hsp_isign[mu][hsp_col[mu][i%2]]%3==0))  ||
	(f==1 && (hsp_isign[mu][hsp_col[mu][i%2]]%3!=0)))
      fadd_isign(r[0], r[1], 
		 hsp_isign[mu][hsp_col[mu][i%2]],
		 fpreg_kappa1);
    else
      fadd_isign(r[0], r[1], 
		 3-hsp_isign[mu][hsp_col[mu][i%2]],
		 fpreg_kappa1);
  }
}


// 0<=i<=8
void load_U(int fw, int mu, int i)
{
  if (fw==0)
  {
    fload(fpreg_U[i], reg_load_U_fw, reg_16, true, true, false);
#ifdef DEBUG_PREF_U_FW
    int reg_scratch[1];
    gpregs.get_regs(1, 0, reg_scratch);
    load(reg_scratch[0], st_off_debug_pref+4, 1);
    store(reg_load_U_fw, 0, reg_scratch[0]);
    add_i(reg_scratch[0], reg_scratch[0], 4);
    store(reg_scratch[0], st_off_debug_pref+4, 1);
    gpregs.release_regs(1, reg_scratch);
#endif

  }
  else
  {
    int reg_offs;
    if ((i%3)!=0)
      reg_offs=reg_48;
    else if (i==3 || i==6)
      reg_offs=reg_m80;
    else
      reg_offs=reg_16;
    fload(fpreg_U[i], reg_load_U_bw, reg_offs, true, true, false);
#ifdef DEBUG_PREF_U_BW
    int reg_scratch[1];
    gpregs.get_regs(1, 0, reg_scratch);
    load(reg_scratch[0], st_off_debug_pref+4, 1);
    load(reg_scratch[0], st_off_debug_pref+4, 1);
    store(reg_load_U_bw, 0, reg_scratch[0]);
    add_i(reg_scratch[0], reg_scratch[0], 4);
    store(reg_scratch[0], st_off_debug_pref+4, 1);
    gpregs.release_regs(1, reg_scratch);
#endif


  }
}


void upd_U_mu_slow(int f, int mu)
{
  if (f!=0)
    return;
#ifdef MU_SLOW
  if (mu!=3)
  {
    add(reg_load_U_fw, reg_load_U_fw, reg_Ufw_offs);
    add_i(reg_load_U_fw, reg_load_U_fw, -144);
  }
  else
  {
    sub(reg_load_U_fw, reg_load_U_fw, reg_Ufw_offs);
    sub(reg_load_U_fw, reg_load_U_fw, reg_Ufw_offs);
    sub(reg_load_U_fw, reg_load_U_fw, reg_Ufw_offs);
  }
#endif
}


// i=0..4
void prefetch_U(int fw, int i)
{
  if (fw==0)
  {
    prefetch(reg_load_U_fw, reg_pref_U_fw);
#ifdef DEBUG_PREF_U_FW
    int reg_scratch[2];
    gpregs.get_regs(2, 0, reg_scratch);
    load(reg_scratch[0], st_off_debug_pref, 1);
    add(reg_scratch[1], reg_load_U_fw, reg_pref_U_fw);
    store(reg_scratch[1], 0, reg_scratch[0]);
    add_i(reg_scratch[0], reg_scratch[0], 4);
    store(reg_scratch[0], st_off_debug_pref, 1);
    gpregs.release_regs(2, reg_scratch);
#endif
  }
  else
  {
    comment("here: pref");
    int reg_offs;
    if (i==0)
      reg_offs=0;
    if (i==1 || i==3)
      reg_offs=reg_16;
    if (i==2 || i==4)
      reg_offs=reg_48;

    prefetch(reg_offs, reg_pref_U_bw);
#ifdef DEBUG_PREF_U_BW
     int reg_scratch[2];
    gpregs.get_regs(2, 0, reg_scratch);
    load(reg_scratch[0], st_off_debug_pref, 1);
    add_i(reg_scratch[1], reg_offs, 0);
    add(reg_scratch[1], reg_scratch[1], reg_pref_U_bw);
    store(reg_scratch[1], 0, reg_scratch[0]);
    add_i(reg_scratch[0], reg_scratch[0], 4);
    store(reg_scratch[0], st_off_debug_pref, 1);
    gpregs.release_regs(2, reg_scratch);
#endif
    if (i==0)
      add_i(reg_pref_U_bw, reg_pref_U_bw, 16);
    if (i==2 || i==4)
      add_i(reg_pref_U_bw, reg_pref_U_bw, 64);
  }
}


void upd_load_reg_psi_l()
{
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);

  load_hword_a(reg_scratch[0], 2, reg_nb_l, true);

  mac(reg_load_psi, reg_scratch[0], reg_48);

  gpregs.release_regs(1, reg_scratch);
} 

void upd_pref_reg_psi_l(int d=4*pref_dist)
// Should be called after upd_load_reg_psi_l
{
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);

  load_hword_a(reg_scratch[0], 2*d, reg_nb_l);

  mac(reg_pref_psi_l, reg_scratch[0], reg_48);

  gpregs.release_regs(1, reg_scratch);
} 

void upd_pref_reg_psi_r()
{
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);

  load_hword_a(reg_scratch[0], 2, reg_nb_pref_r, true);

  mac(reg_pref_psi_r, reg_scratch[0], reg_48);

  gpregs.release_regs(1, reg_scratch);
} 


#ifndef SINGLE
void upd_load_reg_psi_rm(int fw, int mu, int ahead=pr_ahead)
{
  int offs;
  if (fw==0)
    offs=4*mu;
  else
    offs=16+8*mu;
  offs+=48*ahead;
  if (fw==0 && mu==0)
    offs+=48;

  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);

  load_hword_a(reg_scratch[0], 2, reg_nb_rm, true);

  mac(reg_load_psi_r, reg_scratch[0], reg_48);

  gpregs.release_regs(1, reg_scratch);
} 

void check_send_fifo(int fw, int mu, int fpr_scr=-1)
{
  char* l=get_label();
  int fpreg_scratch;
  if (fpr_scr==-1)
    fpregs.get_regs(1, fpregs.reg_scratch, &fpreg_scratch);
  else
    fpreg_scratch=fpr_scr;
  send_fifo_check(fw, mu, fpreg_scratch);
  if (fpr_scr==-1)
    fpregs.release_regs(1, &fpreg_scratch);
  mov_cr(cr_bit(bit_send_ok, fw, mu), 0);



//   if (phys_dir[mu]==3) //XXX
//     set_cr(cr_bit(bit_send_ok, fw, mu), true);
} 

void check_recv_fifo(int fw, int mu, int fpr_scr=-1)
{
  char* l=get_label();
  int fpreg_scratch;
  if (fpr_scr==-1)
    fpregs.get_regs(1, fpregs.reg_scratch, &fpreg_scratch);
  else
    fpreg_scratch=fpr_scr;
  recv_fifo_check(fw, mu, fpreg_scratch);
  if (fpr_scr==-1)
    fpregs.release_regs(1, &fpreg_scratch);
  mov_cr(bit_recv_ok, 1);


//   if (phys_dir[mu]==3) //XXX
//     set_cr(bit_recv_ok, true);

} 
#endif


void upd_load_reg_U(int fw, int mu)
{
  if (fw==0)
    return;
  int offs=2+2*mu;
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
  load_hword_a(reg_scratch[0], offs, reg_nb, false);
  mac(reg_load_U_bw, reg_scratch[0], reg_144);
  gpregs.release_regs(1, reg_scratch);
} 

void upd_pref_reg_U(int fw, int mu, int d=pref_dist)
{
  if (fw==0)
    return;
  comment("here: upd");
  int offs=2+2*mu;
  offs+=10*d;
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
  load_hword_a(reg_scratch[0], offs, reg_nb, false);
  mac(reg_pref_U_bw, reg_scratch[0], reg_144);
  gpregs.release_regs(1, reg_scratch);
} 



#ifndef SINGLE

void upd_nb_bit(int fw, int mu)
{
  int offs;
  if (fw==0)
    offs=4*mu;
  else
    offs=16+8*mu;
  offs=0;
  if (mainloop)
//     offs+=48;
    offs+=10;
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
//   load(reg_scratch[0], offs, reg_nb);
//   and_is(reg_scratch[0], reg_scratch[0], rm_recv_mask);
  load_hword_a(reg_scratch[0], offs, reg_nb);
  comment("upd_nb");
  and_i(reg_scratch[0], reg_scratch[0], 1 << (15-mu-4*fw));
  mov_cr(cr_bit(bit_nb, fw, mu), 2, true);
  gpregs.release_regs(1, reg_scratch);
} 


void upd_pr_rem_bit(int fw, int mu, int ahead=pr_ahead+1)
{
  int offs;
  if (fw==0)
    offs=4*mu;
  else
    offs=16+8*mu;
  offs=0;
//   offs+=48*ahead;
  offs+=10*ahead;
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
//   load(reg_scratch[0], offs, reg_nb);
//   and_is(reg_scratch[0], reg_scratch[0], rm_send_mask);
  load_hword_a(reg_scratch[0], offs, reg_nb);
  comment("upd_rm");
  and_i(reg_scratch[0], reg_scratch[0], 1 << (7-mu-4*fw));
  mov_cr(cr_bit(bit_pr_rm, fw, mu), 2, true);
  gpregs.release_regs(1, reg_scratch);
} 

#endif


void fmovX(int dest, int src, bool par=false)
{
  char str[255];
  sprintf(str, "#move  %i <-- %i", dest, src);
  code.add(new asm_code_element(str));
  fmov(dest, src, par);
}


void rotate_regs(int n, int* from, int* to, int scratch)
{
  for (int i=0; i<n; i++)
    if (from[i]==to[i])
      to[i]=-1;
  bool cont=true;
  while (cont)
  {
    cont=false;
    for (int i=0; i<n; i++)
      if (to[i]!=-1)
      {
	bool needed=false;
	for (int j=0; j<n; j++)
	  if (to[j]!=-1 && from[j]==to[i])
	    needed=true;
	if (!needed)
	{
	  cont=true;
	  fmovX(to[i], from[i], true);
	  to[i]=-1;
	}
      }
  }
  int r=0;
  while (r<n)
  {
    if (to[r]==-1)
    {
      r++;
      continue;
    }
    int needed_to=-1;
    for (int i=r+1; i<n; i++)
      if (from[i]==to[r])
	needed_to=i;

    if (needed_to==-1)
    {
      fmovX(to[r], from[r], true);
      r++;
      continue;
    }
    fmovX(scratch, to[r], true);
    int s=r;
    do
    {
      fmovX(to[s], from[s], true);
      to[s]=-1;
      int s1=-1;
      for (int i=r+1; i<n; i++)
      {
	if (to[i]==from[s])
	  s1=i;
      }
      if (s1==-1)
      {
	fmovX(to[needed_to], scratch, true);
	to[needed_to]=-1;
	r++;
	break;
      }
      s=s1;
    }
    while (true);
  }
}

void start_code()
{
  char str[255];
// allocate "permanent" registers
  gpregs.get_regs(1, 0, &reg_nb);
  gpregs.get_regs(1, 0, &reg_nb_rm);
  gpregs.get_regs(1, 0, &reg_nb_l);
  gpregs.get_regs(1, 0, &reg_nb_pref_r);
  gpregs.get_regs(1, 0, &reg_load_psi_r);
  gpregs.get_regs(1, 0, &reg_pref_psi_l);
  gpregs.get_regs(1, 0, &reg_pref_psi_r);
  gpregs.get_regs(1, 0, &reg_load_psi);
  sprintf(str, "ppd %i", reg_load_psi);
  comment(str);
  gpregs.get_regs(1, 0, &reg_load_U_bw);
  gpregs.get_regs(1, 0, &reg_pref_U_fw);
  gpregs.get_regs(1, 0, &reg_pref_U_bw);
  gpregs.get_regs(1, 0, &reg_16);
  gpregs.get_regs(1, 0, &reg_48);
  gpregs.get_regs(1, 0, &reg_144);
  gpregs.get_regs(1, 0, &reg_m80);
  gpregs.get_regs(1, 0, &reg_80);
  gpregs.get_regs(1, 0, &reg_m176);
  gpregs.get_regs(1, 0, &reg_fifo_stat);
  gpregs.get_regs(1, 0, &reg_Ufw_offs);
  gpregs.get_regs(1, 0, &reg_data);

// "temporary":
  int reg_volume;
  gpregs.get_regs(1, 0, &reg_volume);

  int save_regs_size=sizeof(int)*n_gpr+2*sizeof(double)*n_fr+16;
  int hres_size=8*6*16;
  st_off_save_regs=stack.add(save_regs_size);
  st_off_const_i=stack.add(16);
  st_off_kappa=stack.add(16);
  st_off_scratch16=stack.add(16);
#ifdef DEBUG_PREF
  st_off_debug_pref=stack.add(16);
#endif
#ifndef SINGLE
  st_off_recv_stat_addr=stack.add(32);
  st_off_send_stat_addr=stack.add(32);
  st_off_recv_fifo_addr=stack.add(32);
  st_off_send_fifo_addr=stack.add(32);
  st_off_send_fifo4_addr=stack.add(16);
  st_off_recv_fifo4_addr=st_off_send_fifo4_addr+4;
  st_off_send_header=stack.add(96);
#endif
  if (jupp)
  {
    st_off_jupp=stack.add(8*40);
    sprintf(str, "JUPP %i", st_off_jupp);
    code.add(new asm_code_element(str));
  }
  stack_size=stack.get_size();
  code.add(new asm_code_element(".set    SP, 1"));
  sprintf(str, ".set    stsz, %i", stack_size);
  code.add(new asm_code_element(str));
  if (jupp)
    if (dagger)
    {
      code.add(new asm_code_element(".globl  .a_dirac_dg"));
      code.add(new asm_code_element(".a_dirac_dg:	"));
    }
    else
    {
      code.add(new asm_code_element(".globl  .a_dirac"));
      code.add(new asm_code_element(".a_dirac:	"));
    }
  else
    if (dagger)
    {
      code.add(new asm_code_element(".globl  a_dirac_dg"));
      code.add(new asm_code_element("a_dirac_dg:	"));
    }
    else
    {
      code.add(new asm_code_element(".globl  a_dirac"));
      code.add(new asm_code_element("a_dirac:	"));
    }

// setup new stack frame
  code.add(new asm_code_element("        mflr    0"));
  code.add(new asm_code_element("        stw     0, 4(SP)"));
  code.add(new asm_code_element("        stwu    SP, -stsz(SP)"));

  int reg_scratch[3];
  gpregs.get_regs(3, 0, reg_scratch);


// save registers
  for (int i=0; i<n_gpr; i++)
  {
    sprintf(str, "        stw     %2i, %i(SP)", i, st_off_save_regs+4*i);
    code.add(new asm_code_element(str));
  }
  sprintf(str, "        addi    %2i, SP, %i", reg_scratch[0], st_off_save_regs+4*n_gpr-16);
  code.add(new asm_code_element(str));
  sprintf(str, "        li      %2i, 16", reg_scratch[1]);
  code.add(new asm_code_element(str));
  for (int i=0; i<n_fr; i++)
  {
    sprintf(str, "        stfpdux %2i, %2i, %2i", i, reg_scratch[0], reg_scratch[1]);
    code.add(new asm_code_element(str));
  }
  code.add(new asm_code_element("        mfcr    0"));
  sprintf(str, "        stw     %2i, %i(SP)", 0, 640);
  code.add(new asm_code_element(str));

  code.add(new asm_code_element("#kappa"));
  fneg(1, 1);
  fstore_offs(1, st_off_kappa, 1);


  add_i(reg_load_psi_r, reg_Pinx, 0);

  add_i(reg_load_U_bw, reg_load_U_fw, -16);
  add_i(reg_pref_U_bw, reg_load_U_fw, 0);


  load_hi(reg_data, "wilson_data");
  load_lo(reg_data, "wilson_data");

  load(reg_volume, wd_offs_volume, reg_data);


#ifdef MU_SLOW
#ifdef BQCD
//------------------
  load(reg_scratch[0], wd_offs_volh_tot, reg_data);

  shiftl_i(reg_Ufw_offs, reg_scratch[0], 3);
  add(reg_Ufw_offs, reg_Ufw_offs, reg_scratch[0]);
  shiftl_i(reg_Ufw_offs, reg_Ufw_offs, 5);

//------------------
//     load_hi(reg_volume, "volume");
//     load_lo(reg_volume, "volume");
//     load(reg_volume, 0, reg_volume);
//   load(reg_volume, wd_offs_volume, reg_data);
//------------------


#else
  shiftl_i(reg_Ufw_offs, reg_volume, 3);
  add(reg_Ufw_offs, reg_Ufw_offs, reg_volume);
  shiftl_i(reg_Ufw_offs, reg_Ufw_offs, 4);
#endif
#endif





  char* label_oe1=get_label();
  char* label_oe2=get_label();
  char* label_oe3=get_label();
  cmp_i(0, reg_oe, 0);
  jump_c(label_oe1, 2, false);
// even
#ifdef HOPPING
  add(reg_scratch[0], reg_volume, reg_volume);   //2
  add(reg_scratch[0], reg_scratch[0], reg_volume);  //3
  add(reg_scratch[0], reg_scratch[0], reg_scratch[0]);  //6
  add(reg_scratch[0], reg_scratch[0], reg_scratch[0]);  //12
  add(reg_scratch[1], reg_scratch[0], reg_scratch[0]);  //24
  add(reg_scratch[2], reg_scratch[1], reg_scratch[1]);  //48
  add(reg_scratch[0], reg_scratch[2], reg_scratch[2]);  //96
  sub(reg_load_psi_r, reg_load_psi_r, reg_scratch[0]);
#endif


  shiftr_i(reg_volume, reg_volume, 1);

  load(reg_nb, wd_offs_neighb_offs_even, reg_data);
  load(reg_nb_rm, wd_offs_neighb_offs_rm_even, reg_data);
  load(reg_nb_l, wd_offs_neighb_offs_l_even, reg_data);

  jump_c(label_oe3);
  add_label(label_oe1);
  cmp_i(0, reg_oe, 1);
  jump_c(label_oe2, 2, false);
// odd
  add(reg_scratch[0], reg_volume, reg_volume);   //2
  add(reg_scratch[0], reg_scratch[0], reg_volume);  //3
  add(reg_scratch[0], reg_scratch[0], reg_scratch[0]);  //6
  add(reg_scratch[0], reg_scratch[0], reg_scratch[0]);  //12
  add(reg_scratch[1], reg_scratch[0], reg_scratch[0]);  //24
  add(reg_scratch[2], reg_scratch[1], reg_scratch[1]);  //48
  add(reg_scratch[0], reg_scratch[2], reg_scratch[2]);  //96
  add(reg_Pinx, reg_Pinx, reg_scratch[0]);
#ifndef HOPPING
  add(reg_Pout, reg_Pout, reg_scratch[0]);
#endif

#ifndef MU_SLOW
  add(reg_scratch[1], reg_scratch[0], reg_scratch[0]);  //192
  add(reg_scratch[1], reg_scratch[0], reg_scratch[1]);  //288
#else
#ifdef BQCD
  load(reg_scratch[2], wd_offs_volh_tot, reg_data);

//------------------
//   load_hi(reg_volume, "volh_tot");
//     load_lo(reg_volume, "volh_tot");
//     load(reg_volume, 0, reg_volume);
//------------------
  add(reg_scratch[0], reg_scratch[2], reg_scratch[2]);   //2
  add(reg_scratch[1], reg_scratch[0], reg_scratch[0]);  //4
  add(reg_scratch[1], reg_scratch[1], reg_scratch[1]);  //8
  add(reg_scratch[1], reg_scratch[1], reg_scratch[2]);  //9
  add(reg_scratch[1], reg_scratch[1], reg_scratch[1]);  //18
  add(reg_scratch[1], reg_scratch[1], reg_scratch[1]);  //36
  add(reg_scratch[1], reg_scratch[1], reg_scratch[1]);  //72
  add(reg_scratch[1], reg_scratch[1], reg_scratch[1]);  //144


//------------------
//     load_hi(reg_volume, "volume");
//     load_lo(reg_volume, "volume");
//     load(reg_volume, 0, reg_volume);
//------------------

#else
  add(reg_scratch[1], reg_scratch[2], reg_scratch[1]);  //72
#endif
#endif
  add(reg_load_U_fw, reg_load_U_fw, reg_scratch[1]);
  shiftr_i(reg_volume, reg_volume, 1);

  load(reg_nb, wd_offs_neighb_offs_odd, reg_data);
  load(reg_nb_rm, wd_offs_neighb_offs_rm_odd, reg_data);
  load(reg_nb_l, wd_offs_neighb_offs_l_odd, reg_data);


  jump_c(label_oe3);
  add_label(label_oe2);
// even+odd

  if (jupp)
    load(reg_nb, "addr_nb", 2);
  else
  {
    load(reg_nb, wd_offs_neighb_offs, reg_data);
    load(reg_nb_rm, wd_offs_neighb_offs_rm, reg_data);
    load(reg_nb_l, wd_offs_neighb_offs_l, reg_data);
  }

  add_label(label_oe3);

  gpregs.release_regs(1, &reg_oe);

  add_i(reg_nb_pref_r, reg_nb_rm, -2);
  add_i(reg_nb_rm, reg_nb_rm, -2);
  add_i(reg_nb_l, reg_nb_l, -2);


  add_i(reg_load_psi, reg_load_psi_r, -16);
  add_i(reg_pref_psi_l, reg_load_psi_r, 16);
  add_i(reg_pref_psi_r, reg_load_psi_r, 16);





#ifndef MU_SLOW
  load_i(reg_pref_U_fw, 576*pref_dist);
#else
  load_i(reg_pref_U_fw, 144*pref_dist);
#endif
#ifdef DEBUG_PREF
  load_hi(reg_scratch[0], "debug_pref_buffer");
  load_lo(reg_scratch[0], "debug_pref_buffer");
  load(reg_scratch[0], 0, reg_scratch[0]);
  store(reg_scratch[0], st_off_debug_pref, 1);
  load_hi(reg_scratch[0], "debug_pref_lbuffer");
  load_lo(reg_scratch[0], "debug_pref_lbuffer");
  load(reg_scratch[0], 0, reg_scratch[0]);
  store(reg_scratch[0], st_off_debug_pref+4, 1);
#endif

// setup counter register

  sprintf(str, "mtctr   %i", reg_volume);
  code.add(new asm_code_element(str));


// initialise fp constants


  load_i32(reg_scratch[0], 0xbff00000);
  store(reg_scratch[0], st_off_const_i, 1);
  load_i32(reg_scratch[0], 0x0);
  store(reg_scratch[0], st_off_const_i+4, 1);
  load_i32(reg_scratch[0], 0x3ff00000);
  store(reg_scratch[0], st_off_const_i+8, 1);
  load_i32(reg_scratch[0], 0x0);
  store(reg_scratch[0], st_off_const_i+12, 1);



#ifndef SINGLE
// initialize torus hardware addresses
  if (!jupp)
  {
    load_hi(reg_scratch[0], "send_fifo_addr");
    load_lo(reg_scratch[0], "send_fifo_addr");
    for (int i=0; i<6; i++)
    {
      load(reg_scratch[1], 4*i, reg_scratch[0]);
      store(reg_scratch[1], 4*i+st_off_send_fifo_addr, 1);
    }
    load_hi(reg_scratch[0], "recv_fifo_addr");
    load_lo(reg_scratch[0], "recv_fifo_addr");
    for (int i=0; i<6; i++)
    {
      load(reg_scratch[1], 4*i, reg_scratch[0]);
      store(reg_scratch[1], 4*i+st_off_recv_fifo_addr, 1);
    }
    load_hi(reg_scratch[0], "send_stat_addr");
    load_lo(reg_scratch[0], "send_stat_addr");
    for (int i=0; i<6; i++)
    {
      load(reg_scratch[1], 4*i, reg_scratch[0]);
      store(reg_scratch[1], 4*i+st_off_send_stat_addr, 1);
    }
    load_hi(reg_scratch[0], "recv_stat_addr");
    load_lo(reg_scratch[0], "recv_stat_addr");
    for (int i=0; i<6; i++)
    {
      load(reg_scratch[1], 4*i, reg_scratch[0]);
      store(reg_scratch[1], 4*i+st_off_recv_stat_addr, 1);
    }
    load_hi(reg_scratch[0], "send_header");
    load_lo(reg_scratch[0], "send_header");
    for (int i=0; i<24; i++)
    {
      load(reg_scratch[1], 4*i, reg_scratch[0]);
      store(reg_scratch[1], 4*i+st_off_send_header, 1);
    }
    load_hi(reg_scratch[0], "send_fifo4_addr");
    load_lo(reg_scratch[0], "send_fifo4_addr");
    load(reg_scratch[0], 0, reg_scratch[0]);
    store(reg_scratch[0], st_off_send_fifo4_addr, 1);
    load_hi(reg_scratch[0], "recv_fifo4_addr");
    load_lo(reg_scratch[0], "recv_fifo4_addr");
    load(reg_scratch[0], 0, reg_scratch[0]);
    store(reg_scratch[0], st_off_recv_fifo4_addr, 1);
  }
  load(reg_fifo_stat, st_off_send_stat_addr, 1);

#endif
//setup load registers

  load_i(reg_16, 16);
  load_i(reg_48, 48);
  load_i(reg_144, 144);
  load_i(reg_m80, -80);

    
  add_i(reg_Pinx, reg_Pinx, -16);
  if (!dagger)
    add_i(reg_Pout, reg_Pout, 176);
  else
    add_i(reg_Pout, reg_Pout, 144);
  add_i(reg_load_U_fw, reg_load_U_fw, -16);
  for (int i=0; i<pref_dist ;i++)
    for (int mu=0; mu<4; mu++)
    {
      upd_pref_reg_U(1, mu, i);
      for (int j=0; j<5; j++)
	prefetch_U(1, j);
    }


  if (!dagger)
  {
    load_i(reg_m176, -176);
    load_i(reg_80, -112);
  }
  else
  {
    load_i(reg_m176, -112);
    load_i(reg_80, -176);
  }
  if (!dagger)
    add_i(reg_Pinx, reg_Pinx, 192);
  else
    add_i(reg_Pinx, reg_Pinx, 160);

#ifndef SINGLE
  for (int f=0; f<2; f++)
    for (int mu=0; mu<4; mu++)
      upd_nb_bit(f, mu);
#endif

  fpregs.get_regs(3, fpregs.reg_U, fpreg_U);
  fpregs.get_regs(6, fpregs.reg_psi_pr, fpreg_psi_pr);
  fpregs.get_regs(6, fpregs.reg_hres, fpreg_hres);
  for (int i=0; i<6; i++)
    fpreg_hres_orig[i]=fpreg_hres[i];
  fpregs.get_regs(12, fpregs.reg_res, fpreg_res);
  fpregs.get_regs(1, fpregs.reg_res, &fpreg_kappa1);
  load_i(reg_scratch[0], st_off_const_i);
  fload(fpreg_kappa1, reg_scratch[0], 1, true);
  fload_offs(fpreg_kappa1, st_off_kappa, 1);

  upd_load_reg_psi_l();
  for (int d=0; d<4*pref_dist; d++)
  {
    upd_pref_reg_psi_l(d);
    for (int i=0; i<6; i++)
      prefetch_psi_l(i);
  }
  upd_pref_reg_psi_l();

#ifndef SINGLE
  for (int f=0; f<2; f++)
    for (int mu=0; mu<4; mu++)
      upd_pr_rem_bit(f, mu, 0);
  upd_load_reg_psi_rm(0, 0, -1);

  for (int d=0; d<4*pref_dist; d++)
  {
    upd_pref_reg_psi_r();
    for (int i=0; i<6; i++)
      prefetch_psi_r(i);
  }
  upd_pref_reg_psi_r();


      
  int fpreg_scratch[2];
  fpregs.get_regs(2, fpregs.reg_scratch, fpreg_scratch);


// start test xxx

  int fp_scratch;
  fpregs.get_regs(1, 0, &fp_scratch);
  load_i(reg_scratch[1], wd_offs_fs);
  fload(fp_scratch, 0, reg_fifo_stat, true);
  fstore(fp_scratch, reg_scratch[1], reg_data, true);
//   load_byte(reg_scratch[0], st_off_scratch16+send_stat_byte_offs[0][0], 1);
//   load_byte(reg_scratch[1], st_off_scratch16+send_stat_byte_offs[0][1], 1);
//   add(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
//   load_byte(reg_scratch[1], st_off_scratch16+send_stat_byte_offs[0][2], 1);
//   add(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
//   load_byte(reg_scratch[1], st_off_scratch16+send_stat_byte_offs[1][0], 1);
//   add(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
//   load_byte(reg_scratch[1], st_off_scratch16+send_stat_byte_offs[1][1], 1);
//   add(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
//   load_byte(reg_scratch[1], st_off_scratch16+send_stat_byte_offs[1][2], 1);
//   add(reg_scratch[0], reg_scratch[0], reg_scratch[1]);
//   store(reg_scratch[0], wd_offs_test_data, reg_data);

  fpregs.release_regs(1, &fp_scratch);
// end test xxx




  comment("start send");
  for (int a=0; a<pr_ahead; a++)
  {
    for (int f=0; f<2; f++)
      for (int mu=0; mu<4; mu++)
      {
	int f_n=f;
	int mu_n=(mu+1)%4;
	if (mu_n==0)
	  f_n=1-f_n;
	char* l=get_label();
	jump_c(l, cr_bit(bit_pr_rm, f, mu), false);


  	char* label_wait=add_label();
 	check_send_fifo(1-f, mu);

	fload8(fpreg_scratch[0], 
	       st_off_send_header+16*(3*(1-f)+phys_dir[mu]), 1);



   	jump_c(label_wait, cr_bit(bit_send_ok, 1-f, mu), false);

	get_fifo_send_addr(1-f, mu, reg_scratch[0], reg_scratch[1], reg_scratch[2]);
	if (phys_dir[mu]!=3)
	  fifo_send(1-f, mu, fpreg_scratch[0], reg_scratch[0]); // header


	for (int i=0; i<6; i++)
	{
	  load_psi_rem(f, mu, 2*i, fpreg_scratch[0]);
	  load_psi_rem(f, mu, 2*i+1, fpreg_scratch[1]);
	  psi_pr_rem(f, mu, i, fpreg_scratch);
	  fifo_send(1-f, mu, fpreg_scratch[0], reg_scratch[0]);
	}
	if (phys_dir[mu]==3)
	  put_fifo4_send_addr(reg_scratch[1], reg_scratch[2]);
	else
	  fifo_send(1-f, mu, fpreg_scratch[0], reg_scratch[0]);
	upd_load_reg_psi_rm(f_n, mu_n, a);

	add_label(l);

	upd_pr_rem_bit(f, mu, a+1);
      }
  }
  fpregs.release_regs(2, fpreg_scratch);











// start test xxx

//  int fp_scratch;
  fpregs.get_regs(1, 0, &fp_scratch);
  load_i(reg_scratch[1], wd_offs_fs+16);
  fload(fp_scratch, 0, reg_fifo_stat, true);
  fstore(fp_scratch, reg_scratch[1], reg_data, true);

  fpregs.release_regs(1, &fp_scratch);
// end test xxx





  char* label_pr=get_label();
  jump_c(label_pr, cr_bit(bit_nb, 0, 0), true);

#endif

  

  comment("start load psi");
  for (int i=0; i<4; i++)
    load_psi(0, 0, i, 2);
  for (int i=0; i<2; i++)
    psi_pr(0, 0, i, 2);
  for (int i=4; i<6; i++)
    load_psi(0, 0, i);
  
  prefetch_psi_l(0);
  prefetch_psi_l(1);
  prefetch_psi_l(2);
  prefetch_psi_l(3);
  prefetch_psi_l(4);
  prefetch_psi_l(5);

#ifndef SINGLE
  char* label_pr_done=get_label();
  jump_c(label_pr_done);
  add_label(label_pr);


  check_recv_fifo(0, 0, fpreg_psi_pr[0]);
  jump_c(label_pr, bit_recv_ok, false);

  get_fifo_recv_addr(0, 0, reg_scratch[0], reg_scratch[1],
		     reg_scratch[2]);
  if (phys_dir[0]!=3)
    fifo_recv(0, 0, fpreg_psi_pr[0], reg_scratch[0]);  // header
  for (int i=0; i<6; i++)
    fifo_recv(0, 0, fpreg_psi_pr[i], reg_scratch[0]);
  if (phys_dir[0]==3)
    put_fifo4_recv_addr(reg_scratch[1], reg_scratch[2]);
  else
    fifo_recv(0, 0, fpreg_psi_pr[5], reg_scratch[0]);  // dummy

  add_label(label_pr_done);
#endif



  for (int i=0; i<3; i++)
    load_U(0, 0, i);
  for (int i=0; i<3; i++) 
    fpreg_rem_U[i]=fpreg_U[i];
  for (int i=0; i<6; i++) 
    fpreg_rem_psi_pr[i]=fpreg_psi_pr[i];


  gpregs.release_regs(3, reg_scratch);
  gpregs.release_regs(1, &reg_volume);


}

void load_P_loc(int i)
{
#ifndef HOPPING
  if (dagger)
    comment("LOADLOC");
  int reg_offs;
  if (i==0)
    reg_offs=reg_80;
  if (i==1 || i==3 || i==5 || i==7 || i==9 || i==11)
    reg_offs=reg_16;
  if (i==2 || i==4 || i==8 || i==10)
    reg_offs=reg_48;
  if (i==6)
    reg_offs=reg_m176;
  fload(fpreg_hres_orig[i%6], reg_Pinx, reg_offs, true, true);
#endif
}

void add_P_loc(int i)
{
#ifndef HOPPING
  int j=4*((i%6)/2)+2*(1-(i/6))+i%2;

  j=4*((i%6)/2)+2+i%2;
  if (dagger && i<6)
    j-=2;
  fmuladd_s(fpreg_res[j], fpreg_kappa1, fpreg_res[j],
	    fpreg_hres_orig[i%6]);
#endif
}

void store_res(int i)
{


  if (dagger)
    comment("STORE");
  int reg_offs;
  if (i==0)
    reg_offs=reg_80;
  if (i==1 || i==3 || i==5 || i==7 || i==9 || i==11)
    reg_offs=reg_16;
  if (i==2 || i==4 || i==8 || i==10)
    reg_offs=reg_48;
  if (i==6)
    reg_offs=reg_m176;
  if (!dagger || i>5)
    i=4*((i%6)/2)+2+i%2;
  else
    i=4*((i%6)/2)+i%2;
  fstore(fpreg_res[i], reg_Pout, reg_offs, true, true);
}

void end_code()
{
  char str[255];

// store result of last site
  for (int i=6; i<12; i++)
    load_P_loc(i);
  for (int i=6; i<12; i++)
    add_P_loc(i);
  for (int i=6; i<12; i++)
    store_res(i);


  sprintf(str, "        lwz     %2i, %i(SP)", 0, 640);
  code.add(new asm_code_element(str));
  code.add(new asm_code_element("        mtcr    0"));

  int reg_scratch[2];
  gpregs.get_regs(2, 0, reg_scratch);

  sprintf(str, "        addi    %2i, SP, %i", reg_scratch[0], st_off_save_regs+4*n_gpr-16);
  code.add(new asm_code_element(str));
  sprintf(str, "        li      %2i, 16", reg_scratch[1]);
  code.add(new asm_code_element(str));
  for (int i=0; i<n_fr; i++)
  {
    sprintf(str, "        lfpdux   %2i, %2i, %2i", i, reg_scratch[0], reg_scratch[1]);
    code.add(new asm_code_element(str));
  }

  for (int i=0; i<n_gpr; i++)
  {
    sprintf(str, "        lwz     %2i, %i(SP)", i, st_off_save_regs+4*i);
    code.add(new asm_code_element(str));
  }

  code.add(new asm_code_element("        addi    SP, SP, stsz"));
  code.add(new asm_code_element("        lwz     0, 4(SP)"));
  code.add(new asm_code_element("        mtlr    0"));
  code.add(new asm_code_element("        bclr     20, 0"));

  if (jupp)
  {
    code.add(new asm_code_element(".toc"));
    code.add(new asm_code_element("addr_nb:"));
    code.add(new asm_code_element(".tc neighb_offs[TC], neighb_offs\n"));
    code.add(new asm_code_element("addr_send_fifo:"));
    code.add(new asm_code_element(".tc send_fifo[TC], send_fifo\n"));
    code.add(new asm_code_element("addr_recv_fifo:"));
    code.add(new asm_code_element(".tc recv_fifo[TC], recv_fifo\n"));
  }
  gpregs.release_regs(2, reg_scratch);
}


void mult(int fw, int mu, int i)
{
  int hspin=i%2;    // half spinor to be updated
  int row=(i/2)%3;  // row of U
  int pass=(i/6)%2;  // 2 mult. per component
  int col=i/12;     // column of U


  bool add=(col>0) || (pass>0);
  if (mu==3)
    add=true;
  static int count=0;
  bool flip=false;
  if (!dagger && fw==1 && mu==3 && i>=30)
    flip=true;
  if (!flip)
    cplxmul(fpreg_hres[hspin+2*row], fpreg_U[row+3*col],
	    fpreg_psi_pr[hspin+2*col], pass, add, fw==1,
	    (!dagger && fw==0 && mu==1 && col==2 && hspin==0) ||
	    (dagger && fw==0 && mu==1 && col==2 && hspin==1));
  else
    cplxmul(fpreg_res[hspin+4*row+2], fpreg_U[row+3*col],
	    fpreg_psi_pr[hspin+2*col], pass, add, fw==1,
	    (!dagger && fw==0 && mu==1 && col==2 && hspin==0) ||
	    (dagger && fw==0 && mu==1 && col==2 && hspin==1),
	    fpreg_hres[hspin+2*row]);
}

void add_to_res(int f, int mu, int a)
{
  int col=a/4;
  int hspin=(a%4)%2;
  int pass=(a%4)/2;
  if (f==0 && mu==0)
    return;
  if (f==0 && mu==1)
    pass=1-pass;
  if (mu==3)
    return;
  if (f==0 && mu==1 && pass==1)
  {
    if (!dagger)
      fadd_isign(fpreg_res[4*col+2+hspin],
		 fpreg_res[4*col+1-hspin],
		 1, fpreg_kappa1,
		 fpreg_hres[2*col+hsp_col[mu][hspin]]);
    else
      fadd_isign(fpreg_res[4*col+2+hspin],
		 fpreg_res[4*col+1-hspin],
		 2, fpreg_kappa1,
		 fpreg_hres[2*col+hsp_col[mu][hspin]]);
    return;
  }


  if (pass==0)
  {
    if (f==0 && mu==1 && 
	((hspin==0 && !dagger) || (hspin==1 && dagger)))
      fadd_isign(fpreg_res[4*col+hspin], fpreg_hres[2*col+hspin],
		 3, fpreg_kappa1);
    else
      fadd_isign(fpreg_res[4*col+hspin], fpreg_hres[2*col+hspin],
	       0, fpreg_kappa1);
  }
  else
  {
    if (dagger)
      f=1-f;
    if (f==0)
      fadd_isign(fpreg_res[4*col+2+hspin],
		 fpreg_hres[2*col+hsp_col[mu][hspin]],
		 hsp_isign[mu][hspin], fpreg_kappa1);
    else
      fadd_isign(fpreg_res[4*col+2+hspin],
		 fpreg_hres[2*col+hsp_col[mu][hspin]],
		 3-hsp_isign[mu][hspin], fpreg_kappa1);
  }
}



void mult_Upsi(int f, int mu)
{
  int f_n=f;
  int mu_n=(mu+1)%4;
  if (mu_n==0)
    f_n=1-f_n;
  int f_nn=f_n;
  int mu_nn=(mu_n+1)%4;
  if (mu_nn==0)
    f_nn=1-f_nn;
  int save_fpreg_hres[6];
  if (f==0 && mu==0)
    for (int i=0; i<6; i++)
    {
      save_fpreg_hres[i]=fpreg_hres[i];
      fpreg_hres[i]=fpreg_res[4*(i/2)+(i%2)];
    }
  if (mu==3)
    for (int i=0; i<6; i++)
    {
      save_fpreg_hres[i]=fpreg_hres[i];
      if (dagger)
	fpreg_hres[i]=fpreg_res[4*(i/2)+(i%2)+2*f];
      else
	fpreg_hres[i]=fpreg_res[4*(i/2)+(i%2)+2*(1-f)];
    }

#ifndef SINGLE
  const int pass_num=2;
#else
  const int pass_num=1;
#endif
  for (int pass=0; pass<pass_num; pass++)
  {
    comment("start pass");
#ifndef SINGLE
    char* label_psi1,* label_psi2;
    save_regs* sr;
    if (pass==0)
    {
      sr=new save_regs();
      label_psi1=get_label();
      label_psi2=get_label();
      jump_c(label_psi1, cr_bit(bit_nb, f, mu), true);
    }
    if (pass==1)
    {
      sr->restore_regs();
      delete sr;
      jump_c(label_psi2);
      add_label(label_psi1);
    }
#endif

    mult(f, mu,  0);
    if (pass==0)
      load_psi(f, mu, 6, 4);
    mult(f, mu,  1);
    if (pass==0)
      load_psi(f, mu, 7, 4);
    mult(f, mu,  2);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+0);
    mult(f, mu,  3);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+1);
    mult(f, mu,  4);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+2);
    mult(f, mu,  5);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+3);
    if (pass==0)
      psi_pr(f, mu, 2, 4);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+4);
    if (pass==0)
      psi_pr(f, mu, 3, 4);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      load_P_loc(6*(1-f)+5);
    mult(f, mu,  6);
    mult(f, mu,  7);
    fpregs.release_regs(1, fpreg_U+0);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+3);
    char str[200];
    sprintf(str, "crash!! pass %i f %i mu %i", pass, f, mu);
    comment(str);
    load_U(f, mu, 3);

    mult(f, mu,  8);
    if (pass==0)
      load_psi(f, mu, 8);
    mult(f, mu,  9);
    fpregs.release_regs(1, fpreg_U+1);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+4);
    load_U(f, mu, 4);
    mult(f, mu, 10);
    prefetch_U(f, 2);
    if (pass==0)
      load_psi(f, mu, 9);
    mult(f, mu, 11);
    fpregs.release_regs(1, fpreg_U+2);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+5);
    load_U(f, mu, 5);



    if ((f==1 && mu==3) || (f==0 && mu==0))
      for (int i=0; i<6; i++)
	add_P_loc(6*(1-f)+i);

    mult(f, mu, 12);
    if (pass==0)
      load_psi(f, mu, 10, 0);
    mult(f, mu, 13);
    if (pass==0)
      load_psi(f, mu, 11, 0);

    if (pass==0)
      upd_load_reg_psi_l();

    mult(f, mu, 14);

    if (pass==0)
      upd_pref_reg_psi_l();
    else
      upd_pref_reg_psi_r();

    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+0);
    mult(f, mu, 15);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+1);
    mult(f, mu, 16);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+2);
    mult(f, mu, 17);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+3);
    if (pass==0)
      psi_pr(f, mu, 4, 0);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+4);
    if (pass==0)
      psi_pr(f, mu, 5, 0);
    if ((f==1 && mu==3) || (f==0 && mu==0))
      store_res(6*(1-f)+5);
#ifndef SINGLE
    if (pass==1)
      add_label(label_psi2);
#endif
  }

  mult(f, mu, 18);
  mult(f, mu, 19);
  fpregs.release_regs(1, fpreg_U+3);
  fpregs.get_regs(1, fpregs.reg_U, fpreg_U+6);
  load_U(f, mu, 6);
  mult(f, mu, 20);
  prefetch_U(f, 3);

  for (int pass=0; pass<pass_num; pass++)
  {
    comment("start pass2");
#ifndef SINGLE
    char* label_psi1,* label_psi2;
    save_regs* sr;
    if (pass==0)
    {
      sr=new save_regs();
      label_psi1=get_label();
      label_psi2=get_label();
      jump_c(label_psi1, cr_bit(bit_nb, f_n, mu_n), true);
    }
    if (pass==1)
    {
      sr->restore_regs();
      delete sr;
      jump_c(label_psi2);
      add_label(label_psi1);
    }

    int reg_fifo[3];
    gpregs.get_regs(3, 0, reg_fifo);

//test+++
    if (pass==1)
      add_is(reg_fifo[2], 0, 30000);

//test---



    char* label_check=add_label();

    if (pass==1)
      check_recv_fifo(f_n, mu_n, fpreg_psi_pr[0]);




//test+++

    if (pass==1)
    {
      char str[200];
      sprintf(str, "recv f %i mu %i", f_n, mu_n);
      comment(str);

      add_i(reg_fifo[2], reg_fifo[2], -1);
      cmp_i(0, reg_fifo[2], 0);
      char* label_no_abort=get_label();
      jump_c(label_no_abort, 1, true);






    upd_lr();
    for (int i=0; i<32; i++)
      store(i, wd_offs_gpr+4*i, reg_data);
    sprintf(str, "mfspr %i, %i", reg_fifo[0], 0x009);
    code.add(new asm_code_element(str));
    store(reg_fifo[0], wd_offs_spr, reg_data);
    sprintf(str, "mfcr %i", reg_fifo[0]);
    code.add(new asm_code_element(str));
    store(reg_fifo[0], wd_offs_spr+4, reg_data);
    sprintf(str, "mfspr %i, %i", reg_fifo[0], 0x008);
    code.add(new asm_code_element(str));
    store(reg_fifo[0], wd_offs_spr+8, reg_data);

    add_i(reg_fifo[0], reg_data, wd_offs_fpr-16);
    for (int i=0; i<32; i++)
      fstore(i, reg_fifo[0], reg_16, true, true);

    load_i(reg_fifo[0], wd_offs_fs+32);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_fifo[0], reg_data, true);
    load_i(reg_fifo[0], wd_offs_fs+48);
    add_is(reg_fifo_stat, reg_fifo_stat, 1);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_fifo[0], reg_data, true);


    sprintf(str, "bl error_abort");
    code.add(new asm_code_element(str));




    add_label(label_no_abort);
    }
//test---






    if (pass==1)
      jump_c(label_check, bit_recv_ok, false);
    if (pass==1)
    {
      get_fifo_recv_addr(f_n, mu_n, reg_fifo[0],
			 reg_fifo[1], reg_fifo[2]);
      if (phys_dir[mu_n]!=3)
	fifo_recv(f_n, mu_n, fpreg_psi_pr[0], reg_fifo[0]);  // header
    }
#endif
    if (pass==0)
      load_psi(f_n, mu_n, 0);
#ifndef SINGLE
    else
      fifo_recv(f_n, mu_n, fpreg_psi_pr[0], reg_fifo[0]);
#endif
    mult(f, mu, 21);
    fpregs.release_regs(1, fpreg_U+4);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+7);
    load_U(f, mu, 7);
    mult(f, mu, 22);
    if (pass==0)
      load_psi(f_n, mu_n, 1);
#ifndef SINGLE
    else
      fifo_recv(f_n, mu_n, fpreg_psi_pr[1], reg_fifo[0]);
#endif
    mult(f, mu, 23);

    fpregs.release_regs(1, fpreg_U+5);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+8);
    load_U(f, mu, 8);
    prefetch_U(f, 4);
    upd_U_mu_slow(f, mu);
    upd_load_reg_U(f_n, mu_n);
    upd_pref_reg_U(f_n, mu_n);


    mult(f, mu, 24);
    if (pass==0)
      load_psi(f_n, mu_n, 2, 2);
#ifndef SINGLE
    else
      fifo_recv(f_n, mu_n, fpreg_psi_pr[2], reg_fifo[0]);
#endif
    mult(f, mu, 25);
    if (pass==0)
      load_psi(f_n, mu_n, 3, 2);
#ifndef SINGLE
    else
      fifo_recv(f_n, mu_n, fpreg_psi_pr[3], reg_fifo[0]);
#endif
    mult(f, mu, 26);
    if (pass==0)
      prefetch_psi_l(0);
    else
      prefetch_psi_r(0);
    mult(f, mu, 27);
    if (pass==0)
      prefetch_psi_l(1);
    else
      prefetch_psi_r(1);
    mult(f, mu, 28);
    if (pass==0)
      prefetch_psi_l(2);
    else
      prefetch_psi_r(2);
    mult(f, mu, 29);
    if (pass==0)
      prefetch_psi_l(3);
    else
      prefetch_psi_r(3);
    if (pass==0)
    {
      psi_pr(f_n, mu_n, 0, 2);
      psi_pr(f_n, mu_n, 1, 2);
    }
    if (pass==0)
      prefetch_psi_l(4);
    else
      prefetch_psi_r(4);
    mult(f, mu, 30);
    if (pass==0)
      prefetch_psi_l(5);
    else
      prefetch_psi_r(5);
    mult(f, mu, 31);
    fpregs.release_regs(1, fpreg_U+6);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+0);
    load_U(f_n, mu_n, 0);
    mult(f, mu, 32);
    prefetch_U(f_n, 0);
    if (pass==0)
      load_psi(f_n, mu_n, 4);
    mult(f, mu, 33);
    fpregs.release_regs(1, fpreg_U+7);
    fpregs.get_regs(1, fpregs.reg_U, fpreg_U+1);
    load_U(f_n, mu_n, 1);
    mult(f, mu, 34);
    if (pass==0)
      load_psi(f_n, mu_n, 5);
    mult(f, mu, 35);
#ifndef SINGLE
    if (pass==1)
      fifo_recv(f_n, mu_n, fpreg_psi_pr[4], reg_fifo[0]);
    if (pass==1)
      fifo_recv(f_n, mu_n, fpreg_psi_pr[5], reg_fifo[0]);
    if (pass==1)
      if (phys_dir[mu_n]==3)
	put_fifo4_recv_addr(reg_fifo[1], reg_fifo[2]);
      else
	fifo_recv(f_n, mu_n, fpreg_psi_pr[5], reg_fifo[0]);

    if (pass==1)
      add_label(label_psi2);
    gpregs.release_regs(3, reg_fifo);
#endif

  }
  fpregs.release_regs(1, fpreg_U+8);
  fpregs.get_regs(1, fpregs.reg_U, fpreg_U+2);
  load_U(f_n, mu_n, 2);
  prefetch_U(f_n, 1);


  
#ifndef SINGLE
  char strx[300];
  sprintf(strx, "XXX %i %i", f, mu);
  comment(strx);
  add_to_res(f, mu, 0);
//   add_to_res(f, mu, 1);
comment("HERE");
  char* label_rm1=get_label();
  char* label_rm2=get_label();
  jump_c(label_rm1, cr_bit(bit_pr_rm, f, mu), false);
  int fpreg_send[4];
  fpregs.get_regs(4, fpregs.reg_scratch, fpreg_send);
  int reg_scratch[1];
  gpregs.get_regs(1, 0, reg_scratch);
  int reg_fifo[3];
  gpregs.get_regs(3, 0, reg_fifo);

//test+++
  if (phys_dir[mu]==3)
    add_is(reg_fifo[2], 0, 30000);
//test---

  char* label_wait=add_label();
  if (phys_dir[mu]==3)
    check_send_fifo(1-f, mu, fpreg_send[0]);
  else
  {
    load_i(reg_scratch[0], st_off_scratch16);
    if (send_stat_byte_offs[f][phys_dir[mu]]<8)
      fload(fpreg_send[0], 0, reg_fifo_stat, true, false, true);
    else
      fload(fpreg_send[0], 0, reg_fifo_stat, true, false, false);
  }
  if (phys_dir[mu]!=3)
    add_to_res(f, mu, 1);
  if (phys_dir[mu]!=3) //zzz
    fload8(fpreg_send[1], 
	   st_off_send_header+16*(3*(1-f)+phys_dir[mu]), 1);
  if (phys_dir[mu]!=3)
    add_to_res(f, mu, 2);
  if (phys_dir[mu]==3)
  {
//test+++
    add_i(reg_fifo[2], reg_fifo[2], -1);
    cmp_i(0, reg_fifo[2], 0);
    char* label_no_abort=get_label();
    jump_c(label_no_abort, 1, true);




    char str[200];



    upd_lr();
    for (int i=0; i<32; i++)
      store(i, wd_offs_gpr+4*i, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x009);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr, reg_data);
    sprintf(str, "mfcr %i", reg_scratch[0]);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+4, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x008);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+8, reg_data);

    add_i(reg_scratch[0], reg_data, wd_offs_fpr-16);
    for (int i=0; i<32; i++)
      fstore(i, reg_scratch[0], reg_16, true, true);



    load_i(reg_scratch[0], wd_offs_fs+32);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);
    load_i(reg_scratch[0], wd_offs_fs+48);
    add_is(reg_fifo_stat, reg_fifo_stat, 1);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);


    sprintf(str, "bl error_abort");
    code.add(new asm_code_element(str));




    add_label(label_no_abort);
//test---


    jump_c(label_wait, cr_bit(bit_send_ok, 1-f, mu), false);
    add_to_res(f, mu, 1);
comment("HERE2");
    add_to_res(f, mu, 2);
  }
    
  get_fifo_send_addr(1-f, mu, reg_fifo[0], reg_fifo[1], reg_fifo[2]);

comment("HERE3");
  add_to_res(f, mu, 3);

  load_psi_rem(f, mu, 0, fpreg_send[2]);
  add_to_res(f, mu, 4);
  load_psi_rem(f, mu, 1, fpreg_send[3]);
  add_to_res(f, mu, 5);
  if (phys_dir[mu]!=3)
    fifo_send(1-f, mu, fpreg_send[1], reg_fifo[0]); // header
  add_to_res(f, mu, 6);
  load_psi_rem(f, mu, 2, fpreg_hres_orig[0]);
  add_to_res(f, mu, 7);
//   prefetch_psi_r(0);
  load_psi_rem(f, mu, 3, fpreg_hres_orig[1]);
  add_to_res(f, mu, 8);
//   prefetch_psi_r(1);
  load_psi_rem(f, mu, 4, fpreg_hres_orig[2]);
  psi_pr_rem(f, mu, 0, fpreg_send+2, fpreg_send+3);
  load_psi_rem(f, mu, 5, fpreg_send[1]);
  psi_pr_rem(f, mu, 1, fpreg_hres_orig+0, fpreg_hres_orig+1);
  load_psi_rem(f, mu, 6, fpreg_send[3]);
  add_to_res(f, mu, 9);
//   prefetch_psi_r(2);
  load_psi_rem(f, mu, 7, fpreg_hres_orig[3]);
  psi_pr_rem(f, mu, 2, fpreg_hres_orig+2, fpreg_send+1);
  load_psi_rem(f, mu, 8, fpreg_hres_orig[1]);
//   prefetch_psi_r(3);
  add_to_res(f, mu,10);
  load_psi_rem(f, mu, 9, fpreg_send[1]);
  add_to_res(f, mu,11);
//   prefetch_psi_r(4);
  load_psi_rem(f, mu,10, fpreg_hres_orig[4]);
  psi_pr_rem(f, mu, 3, fpreg_send+3, fpreg_hres_orig+3);
  load_psi_rem(f, mu,11, fpreg_hres_orig[5]);
//   prefetch_psi_r(5);
  if (phys_dir[mu]!=3)
    fstore8x(fpreg_send[0], reg_scratch[0], 1, true);

  fifo_send(1-f, mu, fpreg_send[2], reg_fifo[0]);
  psi_pr_rem(f, mu, 4, fpreg_hres_orig+1, fpreg_send+1);
  fifo_send(1-f, mu, fpreg_hres_orig[0], reg_fifo[0]);
  psi_pr_rem(f, mu, 5, fpreg_hres_orig+4, fpreg_hres_orig+5);
  fifo_send(1-f, mu, fpreg_hres_orig[2], reg_fifo[0]);
  fifo_send(1-f, mu, fpreg_send[3], reg_fifo[0]);
  if (phys_dir[mu]!=3)
    load_byte(reg_scratch[0], st_off_scratch16+(send_stat_byte_offs[f][phys_dir[mu]]%8), 1);
  fifo_send(1-f, mu, fpreg_hres_orig[1], reg_fifo[0]);
  fifo_send(1-f, mu, fpreg_hres_orig[4], reg_fifo[0]);
  upd_load_reg_psi_rm(f_n, mu_n);
//   upd_pref_reg_psi_r();
  if (phys_dir[mu]!=3)
    cmp_i(0, reg_scratch[0], fifo_size-12);
  if (phys_dir[mu]!=3)
    fifo_send(1-f, mu, fpreg_hres_orig[4], reg_fifo[0]);
  else
    put_fifo4_send_addr(reg_fifo[1], reg_fifo[2]);


  fpregs.release_regs(4, fpreg_send);

  if (phys_dir[mu]!=3)
  {
    char s[200];
    sprintf(s, "mu %i f %i", mu , f);
    comment(s);
    jump_c(label_rm2, 0, true);


    add_is(reg_fifo[2], 0, 30000);
    char* idle_loop=add_label();
    add_i(reg_fifo[2], reg_fifo[2], -1);
    cmp_i(0, reg_fifo[2], 0);
    char* label_no_abort=get_label();
    jump_c(label_no_abort, 1, true);
    
    char str[200];


//-----------
    upd_lr();
    for (int i=0; i<32; i++)
      store(i, wd_offs_gpr+4*i, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x009);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr, reg_data);
    sprintf(str, "mfcr %i", reg_scratch[0]);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+4, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x008);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+8, reg_data);

    add_i(reg_scratch[0], reg_data, wd_offs_fpr-16);
    for (int i=0; i<32; i++)
      fstore(i, reg_scratch[0], reg_16, true, true);


    load_i(reg_scratch[0], wd_offs_fs+32);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);
    load_i(reg_scratch[0], wd_offs_fs+48);
    add_is(reg_fifo_stat, reg_fifo_stat, 1);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);



    sprintf(str, "bl error_abort");
    code.add(new asm_code_element(str));
//---------



    add_label(label_no_abort);

    load_i(reg_scratch[0], st_off_scratch16);
    if (send_stat_byte_offs[f][phys_dir[mu]]<8)
      fload(fpreg_send[1], 0, reg_fifo_stat, true, false, true);
    else
      fload(fpreg_send[1], 0, reg_fifo_stat, true, false, false);
    fstore8x(fpreg_send[1], reg_scratch[0], 1, true);



    load_byte(reg_scratch[0], st_off_scratch16+(send_stat_byte_offs[f][phys_dir[mu]]%8), 1);
    cmp_i(0, reg_scratch[0], fifo_size-9);


    jump_c(idle_loop, 0, false);
    jump_c(label_rm2);
  }
  else
    jump_c(label_rm2);

  add_label(label_rm1);
  gpregs.release_regs(3, reg_fifo);

  add_to_res(f, mu, 1);
  add_to_res(f, mu, 2);
  add_to_res(f, mu, 3);
  add_to_res(f, mu, 4);
  add_to_res(f, mu, 5);
  add_to_res(f, mu, 6);
  add_to_res(f, mu, 7);
  add_to_res(f, mu, 8);
  add_to_res(f, mu, 9);
  add_to_res(f, mu,10);
  add_to_res(f, mu,11);

  add_label(label_rm2);
//   upd_load_reg_psi_rm(f_n, mu_n);
  upd_pr_rem_bit(f, mu);
  upd_nb_bit(f, mu);
  gpregs.release_regs(1, reg_scratch);
#else
  for (int i=0; i<12; i++)
    add_to_res(f, mu, i);

#endif

  if ((f==0 && mu==0) || mu==3)
    for (int i=0; i<6; i++)
      fpreg_hres[i]=save_fpreg_hres[i];


}




void rotate()
{

  char sstr[2000];
  sprintf(sstr, "# U: %i->%i   %i->%i   %i->%i   %i->%i   %i->%i   %i->%i   %i->%i   %i->%i   %i->%i",
	  fpreg_U[0], fpreg_rem_U[0], 
	  fpreg_U[1], fpreg_rem_U[1], 
	  fpreg_U[2], fpreg_rem_U[2], 
	  fpreg_U[3], fpreg_rem_U[3], 
	  fpreg_U[4], fpreg_rem_U[4], 
	  fpreg_U[5], fpreg_rem_U[5], 
	  fpreg_U[6], fpreg_rem_U[6], 
	  fpreg_U[7], fpreg_rem_U[7], 
	  fpreg_U[8], fpreg_rem_U[8]);
  code.add(new asm_code_element(sstr));
  sprintf(sstr, "# P: %i->%i   %i->%i   %i->%i   %i->%i   %i->%i   %i->%i",
	  fpreg_psi_pr[0], fpreg_rem_psi_pr[0],
	  fpreg_psi_pr[1], fpreg_rem_psi_pr[1],
	  fpreg_psi_pr[2], fpreg_rem_psi_pr[2],
	  fpreg_psi_pr[3], fpreg_rem_psi_pr[3],
	  fpreg_psi_pr[4], fpreg_rem_psi_pr[4],
	  fpreg_psi_pr[5], fpreg_rem_psi_pr[5]);
  code.add(new asm_code_element(sstr));
  int fpreg_from[15], fpreg_to[15];
  for (int i=0; i<3; i++)
  {
    fpreg_from[i]=fpreg_U[i];
    fpreg_to[i]=fpreg_rem_U[i];
  }
  for (int i=0; i<6; i++)
  {
    fpreg_from[3+i]=fpreg_psi_pr[i];
    fpreg_to[3+i]=fpreg_rem_psi_pr[i];
  }


  fpregs.release_regs(3, fpreg_U);
  fpregs.release_regs(6, fpreg_psi_pr);
  fpregs.get_these_regs(3, fpregs.reg_U, fpreg_rem_U);
  fpregs.get_these_regs(6, fpregs.reg_psi_pr, fpreg_rem_psi_pr);


  int fpreg_scratch;
  fpregs.get_regs(1, fpregs.reg_scratch, &fpreg_scratch);
  rotate_regs(9, fpreg_from, fpreg_to, fpreg_scratch);
  fpregs.release_regs(1, &fpreg_scratch);

}


// bg? lat_x lat_y lat_z lat_t

int main(int argc, char** argv)
{
  char str[255];
  if (argc!=6)
  {
    printf("usage: cr_asm bg? dir_x dir_y dir_z dir_t\n");
    exit(1);
  }
  jupp=(atoi(argv[1])==0);
  phys_dir[0]=atoi(argv[2]);
  phys_dir[1]=atoi(argv[3]);
  phys_dir[2]=atoi(argv[4]);
  phys_dir[3]=atoi(argv[5]);

  for (int dg=0; dg<2; dg++)
  {
    code.init();
    fpregs.reset();
    gpregs.reset();

    gpregs.get_these_regs(1, 0, &reg_0);
    gpregs.get_these_regs(1, 0, &reg_stack);
    gpregs.get_these_regs(1, 0, &reg_toc);
    gpregs.get_these_regs(1, 0, &reg_load_U_fw);
    gpregs.get_these_regs(1, 0, &reg_Pinx);
    gpregs.get_these_regs(1, 0, &reg_Pout);
    gpregs.get_these_regs(1, 0, &reg_oe);


    dagger=dg==1;

    for (int i=0; i<9; i++)
      fpreg_U[i]=-1;
    for (int i=0; i<6; i++)
      fpreg_psi_pr[i]=-1;
    for (int i=0; i<6; i++)
      fpreg_hres[i]=-1;


    start_code();

    if (dagger)
    {
      if (jupp)
	code.add(new asm_code_element("bl d_loop_dg"));
      code.add(new asm_code_element("d_loop_dg:"));
    }
    else
    {
      if (jupp)
	code.add(new asm_code_element("bl d_loop"));
      code.add(new asm_code_element("d_loop:"));
    }

    mainloop=true;

    for (int f=0; f<2; f++)
      for (int mu=0; mu<4; mu++)
	mult_Upsi(f, mu);


#ifndef NO_SYNC
// synchronize in order to avoid deadlocks
    int reg_scratch[4];
    gpregs.get_regs(4, 0, reg_scratch);



//test+++
    add_is(reg_scratch[3], 0, 30000);

//test---

 
    load(reg_scratch[0], st_off_send_fifo4_addr, 1);
    load(reg_scratch[1], st_off_recv_fifo4_addr, 1);
    load(reg_scratch[2], 8, reg_scratch[0]); // my counter
    load(reg_scratch[1], 8, reg_scratch[1]); // neighbour's counter
    add_i(reg_scratch[2], reg_scratch[2], 1);
    store(reg_scratch[2], 8, reg_scratch[0]); // update my counter
    sub(reg_scratch[0], reg_scratch[2], reg_scratch[1]);
    cmp_i(0, reg_scratch[0], max_diff+1);
    char* label_cont=get_label();
    char* label_sync_loop=add_label();
    jump_c(label_cont, 0, true);














    char* label_check=add_label();




//test+++

    comment("syncloop");
    
    add_i(reg_scratch[3], reg_scratch[3], -1);
    cmp_i(0, reg_scratch[3], 0);
    char* label_no_abort=get_label();
    jump_c(label_no_abort, 1, true);
    

    upd_lr();
    for (int i=0; i<32; i++)
      store(i, wd_offs_gpr+4*i, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x009);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr, reg_data);
    sprintf(str, "mfcr %i", reg_scratch[0]);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+4, reg_data);
    sprintf(str, "mfspr %i, %i", reg_scratch[0], 0x008);
    code.add(new asm_code_element(str));
    store(reg_scratch[0], wd_offs_spr+8, reg_data);

    add_i(reg_scratch[0], reg_data, wd_offs_fpr-16);
    for (int i=0; i<32; i++)
      fstore(i, reg_scratch[0], reg_16, true, true);

    load_i(reg_scratch[0], wd_offs_fs+32);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);
    load_i(reg_scratch[0], wd_offs_fs+48);
    add_is(reg_fifo_stat, reg_fifo_stat, 1);
    fload(31, 0, reg_fifo_stat, true);
    fstore(31, reg_scratch[0], reg_data, true);


    sprintf(str, "bl error_abort");
    code.add(new asm_code_element(str));




    add_label(label_no_abort);

//test---












    load(reg_scratch[1], st_off_recv_fifo4_addr, 1);
    load(reg_scratch[1], 8, reg_scratch[1]); // neighbour's counter
    sub(reg_scratch[0], reg_scratch[2], reg_scratch[1]);
    cmp_i(0, reg_scratch[0], max_diff+1);
    jump_c(label_sync_loop);

    add_label(label_cont);
    gpregs.release_regs(4, reg_scratch);
#endif

    if (!dagger)
      load_i(reg_80, 80);
    else
      load_i(reg_80, 16);
    rotate();
    add_i(reg_nb, reg_nb, 10);


    if (dagger)
      if (jupp)
	sprintf(str, "bclr  17, 31");
      else
	sprintf(str, "bc  17, 31, d_loop_dg");
    else
      if (jupp)
	sprintf(str, "bclr  17, 31");
      else
	sprintf(str, "bc  17, 31, d_loop");
    code.add(new asm_code_element(str));
    mainloop=false;

 

    end_code();

    code.listing();
  }
  printf("# max gpr: %i\n", gpregs.get_max_reg());
}
