//==============================================================================
//
// d/wilson_d520/bgp_wilson.cc
//
//------------------------------------------------------------------------------
//
// Copyright (C) 2010 Thomas Streuer 
//
// This file is part of BQCD -- Berlin Quantum ChromoDynamics program
//
// BQCD is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// BQCD is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with BQCD.  If not, see <http://www.gnu.org/licenses/>.
//
//------------------------------------------------------------------------------
#include "bgp_wilson.h"
#include "common.h"

#include <vector>
#include <sstream>
#include <string>

using std::istringstream;
using std::ostringstream;
using std::string;
using std::vector;

bgp_wilson_t::bgp_wilson_t(bool dagger, bool single, bool debug)
  : gpr(32), fpr(32), cr(8),
    stack(), label(),
    dagger_(dagger),
    single_(single),
    debug_(debug),
    dont_upd_U  (false),
    dont_upd_psi(false),
    dont_upd_r  (false),
    pref_psi_kind(pref_no)
{
  name = string("a_dirac");
  if (single_)
    name += string("_sng");
  if (dagger_)
    name += string("_dg");
  label.base() = name;
  nb_array_offs = 8 * 18 * sizeof(double) + 8 * sizeof(uint16_t); //PREC
  alloc_regs();
  header();
  init_stack();
  init_regs();
  wait_loop_label = label.get_label();

  prolog();
  string loop_label("");
  add(label.set_label(loop_label));

  for (unsigned fw=0; fw<2; ++fw)
    for (unsigned mu=0; mu<4; ++mu)
      mult_Upsi(fw, mu);


  for (unsigned i=0; i<9; ++i)
    cmp_counters(i);
  add(bgp_code_element_t::jump_cond(wait_loop_label, cr_0, bgp_code_element_t::cr_field_lt, 0, true));
  cr.release_reg(cr_0);

  add(bgp_code_element_t::jump_ctr(loop_label));
  string label_finish = label.get_label();
  add(bgp_code_element_t::jump_cond(label_finish, cr_it, 0, 1));

  if (T0_LAST)
  {
    string label_f2 = label.get_label();
    add(bgp_code_element_t::jump_cond(label_f2, cr_it, 1, 1));
    add(bgp_code_element_t::crset(cr_it, 1, 1));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_0, 1));
    add(bgp_code_element_t::mtctr(gpr_store_r));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_sp, st_offs_res0 + (dagger_? 144 : 176)));//PREC
    add(bgp_code_element_t::jump(loop_label));

    add(label.set_label(label_f2));
    add(bgp_code_element_t::crset(cr_it, 0, 1));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_ctr3));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, -1));
    add(bgp_code_element_t::mtctr(gpr_store_r));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_res_addr));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, dagger_? 192-48 : 192-16)); //PREC
    add(bgp_code_element_t::jump(loop_label));
  }
  else
  {
    add(bgp_code_element_t::crset(cr_it, 0, 1));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_vol));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, -1));
    add(bgp_code_element_t::mtctr(gpr_store_r));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_res_addr));
    if (!dagger_)
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, 192-16)); //PREC
    else
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, 192-48)); //PREC
    add(bgp_code_element_t::jump(loop_label));
  }
  add(label.set_label(label_finish));
  
  epilog();
  finish();
  wait_counters();
  if (debug_)
    debug_regs();
}

void bgp_wilson_t::header()
{
  add(asm_code_element_t(string(".globl\t") + name));
  add(asm_code_element_t(name + string(":")));
}

void bgp_wilson_t::alloc_regs()
{
  vector<unsigned> fixed_gprs;
  fixed_gprs.push_back(gpr_16);
  fixed_gprs.push_back(gpr_sp);
  fixed_gprs.push_back(gpr_psi_base);
  fixed_gprs.push_back(gpr_store_r);
  fixed_gprs.push_back(gpr_data);
  gpr.get_these_regs(fixed_gprs);
  gpr_load_U        = gpr.get_reg();
  gpr_nb            = gpr.get_reg();
  gpr_nb_idx        = gpr.get_reg();
  gpr_m16           = gpr.get_reg();
  gpr_32            = gpr.get_reg();
  gpr_48            = gpr.get_reg();
  gpr_80            = gpr.get_reg();
  gpr_m80           = gpr.get_reg();
  gpr_96            = gpr.get_reg();
  gpr_m96           = gpr.get_reg();
  gpr_psi_nb_buffer = gpr.get_reg();
  gpr_counter_base  = gpr.get_reg();
  gpr.get_regs(4, gpr_load_psi);
  gpr.get_regs(4, gpr_counter);
  gpr.get_regs(4, gpr_counter_hw);
  if (pref_psi_kind != pref_no)
    gpr_pref_psi  = gpr.get_reg();
  fpr_i         = fpr.get_reg();
  ostringstream oss;
  oss << gpr_load_U;
  add(bgp_code_element_t::comment(string("gpr load U: " + oss.str())));
  oss.str("");
  oss << gpr_nb;
  add(bgp_code_element_t::comment(string("gpr nb: " + oss.str())));
  oss.str("");
  oss << gpr_nb_idx;
  add(bgp_code_element_t::comment(string("gpr nb_idx: " + oss.str())));
  oss.str("");
  oss << gpr_16;
  add(bgp_code_element_t::comment(string("gpr 16: " + oss.str())));
  oss.str("");
  oss << gpr_m16;
  add(bgp_code_element_t::comment(string("gpr -16: " + oss.str())));
  oss.str("");
  oss << gpr_32;
  add(bgp_code_element_t::comment(string("gpr 32: " + oss.str())));
  oss.str("");
  oss << gpr_48;
  add(bgp_code_element_t::comment(string("gpr 48: " + oss.str())));
  oss.str("");
  oss << gpr_80;
  add(bgp_code_element_t::comment(string("gpr 80: " + oss.str())));
  oss.str("");
  oss << gpr_m80;
  add(bgp_code_element_t::comment(string("gpr -80: " + oss.str())));
  oss.str("");
  oss << gpr_96;
  add(bgp_code_element_t::comment(string("gpr 96: " + oss.str())));
  oss.str("");
  oss << gpr_m96;
  add(bgp_code_element_t::comment(string("gpr -96: " + oss.str())));
  oss.str("");
  oss << gpr_psi_nb_buffer;
  add(bgp_code_element_t::comment(string("gpr psi_nb_buffer: " + oss.str())));
  for (unsigned mu=0; mu<4; ++mu)
  {
    oss.str("");
    oss << "gpr load psi[" << mu << "] : " << gpr_load_psi[mu];
    add(bgp_code_element_t::comment(oss.str()));
  }
  for (unsigned i=0; i<4; ++i)
  {
    oss.str("");
    oss << "gpr counter[" << i << "] : " << gpr_counter[i];
    add(bgp_code_element_t::comment(oss.str()));
  }
  for (unsigned i=0; i<4; ++i)
  {
    oss.str("");
    oss << "gpr counter_hw[" << i << "] : " << gpr_counter_hw[i];
    add(bgp_code_element_t::comment(oss.str()));
  }
  if (pref_psi_kind != pref_no)
  {
    oss.str("");
    oss << gpr_pref_psi;
    add(bgp_code_element_t::comment(string("gpr pref psi: " + oss.str())));
  }
  oss.str("");
  oss << gpr.max_reg();
  add(bgp_code_element_t::comment(string("max gpr: " + oss.str())));
  oss.str("");
  oss << fpr_i;
  add(bgp_code_element_t::comment(string("fpr i: " + oss.str())));
  fpr.get_regs(5, fpr_U);
  fpr.get_regs(8, fpr_psi_pr);
  fpr.get_regs(6, fpr_hres);
  fpr.get_regs(12, fpr_res);

  vector<unsigned> preserved_crs;   //cr fields 2-4 must be preserved --> don't use them
  preserved_crs.push_back(2);
  preserved_crs.push_back(3);
  preserved_crs.push_back(4);
  preserved_crs.push_back(cr_0);   // reserve cr_0 for later
  cr.get_these_regs(preserved_crs);
  cr_it      = cr.get_reg();
  cr.release_reg(cr_0);

  if (pref_psi_kind == pref_load)
    gpr.get_regs(3, gpr_pref_psi_d);

  st_offs_save_regs = stack.add(4*32+16*32+16); // 32 gpr, 32 dh-fpr, 
  st_offs_scratch32 = stack.add(32); // 32 bytes scratch
  st_offs_res0      = stack.add(384); // PREC res. it 0
  st_offs_res_addr  = stack.add(16); // should be multiple of 16B (stack alignment)
  st_offs_ctr3      = st_offs_res_addr + 4;
  st_offs_vol       = st_offs_res_addr + 8;
}

void bgp_wilson_t::init_stack()
{
  stack_sz = stack.get_size();
  add(bgp_code_element_t::mflr(gpr_0));
  add(bgp_code_element_t::store(gpr_0, gpr_sp, 4));
  add(bgp_code_element_t::store(gpr_sp, gpr_sp, -int(stack_sz), true));
// save registers
  for (unsigned i=0; i<32; ++i)
    add(bgp_code_element_t::store(i, gpr_sp, st_offs_save_regs+4*i));
  vector<unsigned> reg_scratch;
  gpr.get_regs(2, reg_scratch);
  add(bgp_code_element_t::add_i(reg_scratch[0], gpr_0, 16));
  add(bgp_code_element_t::add_i(reg_scratch[1], gpr_sp, st_offs_save_regs+4*32-16));
  for (unsigned i=0; i<32; ++i)
    add(bgp_code_element_t::fstore(i, reg_scratch[1], reg_scratch[0], true, true));
  gpr.release_regs(reg_scratch);
}

void bgp_wilson_t::init_regs()
{
  vector<unsigned> reg_scratch;
  gpr.get_regs(2, reg_scratch);

  add(bgp_code_element_t::load(gpr_data, gpr_data, 0));    // odd/even
  add(bgp_code_element_t::mul_i(gpr_data, gpr_data, 2));

  add(bgp_code_element_t::load_hi(reg_scratch[0], 0, "wilson_data"));
  add(bgp_code_element_t::load_lo(reg_scratch[0], reg_scratch[0], "wilson_data"));

  add(bgp_code_element_t::load(reg_scratch[1], reg_scratch[0],
			       offsetof(global_wilson_data_t, mod2_ctr)));
  add(bgp_code_element_t::add(gpr_data, gpr_data, reg_scratch[1]));
  add(bgp_code_element_t::xor_i(reg_scratch[1], reg_scratch[1], 1));
  add(bgp_code_element_t::store(reg_scratch[1], reg_scratch[0],
			       offsetof(global_wilson_data_t, mod2_ctr)));

  add(bgp_code_element_t::mul_i(gpr_data, gpr_data, sizeof(wilson_data_t)));
  add(bgp_code_element_t::add_i(gpr_data, gpr_data,
				offsetof(global_wilson_data_t, w_data)));
  add(bgp_code_element_t::add(gpr_data, reg_scratch[0], gpr_data));
  add(bgp_code_element_t::add_i(gpr_psi_base, gpr_psi_base, -16));

  add(bgp_code_element_t::load(gpr_nb, gpr_data, offsetof(wilson_data_t, neighb)));
  add(bgp_code_element_t::load(gpr_load_U, gpr_data, offsetof(wilson_data_t, U)));
  add(bgp_code_element_t::load(gpr_psi_nb_buffer, gpr_data, offsetof(wilson_data_t, comm_buffer)));
  add(bgp_code_element_t::load(gpr_counter_base, gpr_data, offsetof(wilson_data_t, rec_counter_hw)));

  add(bgp_code_element_t::add_i(gpr_16, gpr_0, 16));
  add(bgp_code_element_t::add_i(gpr_m16, gpr_0, -16));
  add(bgp_code_element_t::add_i(gpr_32, gpr_0, 32));
  add(bgp_code_element_t::add_i(gpr_48, gpr_0, 48));
  add(bgp_code_element_t::add_i(gpr_80, gpr_0, 80));
  add(bgp_code_element_t::add_i(gpr_m80, gpr_0, -80));
  add(bgp_code_element_t::add_i(gpr_96, gpr_0, 96));
  add(bgp_code_element_t::add_i(gpr_m96, gpr_0, -96));
  add(bgp_code_element_t::crset(cr_it, 0, 0));
  add(bgp_code_element_t::crset(cr_it, 1, 0));
  for (unsigned i=0; i<4; ++i)
    add(bgp_code_element_t::load(gpr_counter[i], gpr_data, offsetof(wilson_data_t, ctr_init) +
				 i * sizeof(uint32_t)));

  add(bgp_code_element_t::load(reg_scratch[0], gpr_data, offsetof(wilson_data_t, volh3)));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_ctr3));
  add(bgp_code_element_t::store(gpr_store_r, gpr_sp, st_offs_res_addr));
  if (T0_LAST)
  {
    add(bgp_code_element_t::mul_i(reg_scratch[0], reg_scratch[0], 192)); // PREC
    add(bgp_code_element_t::add(gpr_store_r, gpr_store_r, reg_scratch[0]));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, dagger_? -48 : -16));
  }
  else
  {  
    if (!dagger_)
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_sp, st_offs_res0+176));//PREC  // during first iteration, save results on stack and put into correct place later
    else
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_sp, st_offs_res0+144));//PREC
  }
  add(bgp_code_element_t::add_i(gpr_load_U, gpr_load_U, -32));
  add(bgp_code_element_t::add_i(gpr_nb, gpr_nb, -nb_array_offs));
  add(bgp_code_element_t::add_i(gpr_psi_nb_buffer, gpr_psi_nb_buffer, -16));
  if (pref_psi_kind != pref_no)
  {
    // +16 corrects for shift in gpr_psi_base
   add(bgp_code_element_t::load(gpr_pref_psi, gpr_data, offsetof(wilson_data_t, pref_offset) + 16));
   add(bgp_code_element_t::add(gpr_pref_psi, gpr_pref_psi, gpr_psi_base));
  }

// initialize counter
  add(bgp_code_element_t::load(reg_scratch[0], gpr_data, offsetof(wilson_data_t, volh)));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_vol));
  add(bgp_code_element_t::load(reg_scratch[1], gpr_data, offsetof(wilson_data_t, volh3)));
  if (T0_LAST)
  {
    add(bgp_code_element_t::sub(reg_scratch[0], reg_scratch[0], reg_scratch[1]));
    add(bgp_code_element_t::mtctr(reg_scratch[0]));
  }
  else
  {
    add(bgp_code_element_t::add_i(reg_scratch[0], gpr_0, 1));
    add(bgp_code_element_t::mtctr(reg_scratch[0]));
  }

// initialize fp constants

  add(bgp_code_element_t::load_i32(reg_scratch[0], 0xbff00000));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_scratch32));
  add(bgp_code_element_t::load_i32(reg_scratch[0], 0x0));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_scratch32+4));
  add(bgp_code_element_t::load_i32(reg_scratch[0], 0x3ff00000));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_scratch32+8));
  add(bgp_code_element_t::load_i32(reg_scratch[0], 0x0));
  add(bgp_code_element_t::store(reg_scratch[0], gpr_sp, st_offs_scratch32+12));


  add(bgp_code_element_t::load_i(reg_scratch[0], st_offs_scratch32));

  add(bgp_code_element_t::fload(fpr_i, reg_scratch[0], gpr_sp, true));

  gpr.release_regs(reg_scratch);
  gpr.release_reg(gpr_data);
}


void bgp_wilson_t::finish()
{
// restore registers
  vector<unsigned> reg_scratch;
  gpr.get_regs(2, reg_scratch);
  add(bgp_code_element_t::add_i(reg_scratch[0], gpr_0, 16));
  add(bgp_code_element_t::add_i(reg_scratch[1], gpr_sp, st_offs_save_regs+4*32-16));
  for (unsigned i=0; i<32; ++i)
    add(bgp_code_element_t::fload(i, reg_scratch[1], reg_scratch[0], true, true));
  gpr.release_regs(reg_scratch);
  for (unsigned i=0; i<32; ++i)
    add(bgp_code_element_t::load(i, gpr_sp, st_offs_save_regs+4*i));
// restore stack pointer
  add(bgp_code_element_t::add_i(gpr_sp, gpr_sp, stack_sz));
// restore gpr 0
  add(bgp_code_element_t::load(gpr_0, gpr_sp, 4));
  add(bgp_code_element_t::mtlr(gpr_0));
  add(asm_code_element_t("        bclr     20, 0"));
}

unsigned bgp_wilson_t::fpr_U_idx(unsigned fw, unsigned mu, unsigned i)
{

  if (i == 0 || i == 4)
    return 0;
  if (i == 1 || i == 6)
    return 1;
  if (i == 3 || i == 7)
    return 3;
  if (i == 2 || i == 8)
    return 4 - 2 * (mu % 2);
  if (i == 5)
    return 2 + 2 * (mu % 2);
  return (unsigned) -1;
}

// i = 0..8
void bgp_wilson_t::load_U(unsigned fw, unsigned mu, unsigned i)
{
  const unsigned iperm[9] = {0, 2, 1, 3, 5, 4, 6, 8, 7};
  i = iperm[i];
  unsigned reg_offs = i % 3 == 1? gpr_m16 : gpr_32;
  if (fw == 0 && mu == 0 && i == 0)
    reg_offs = gpr_48;
  unsigned dest = fpr_U_idx(fw, mu, i);
  ostringstream comment;
  comment << "load U fw/mu " << fw << "/" << mu << "   i " << i;
  add(bgp_code_element_t::fload(fpr_U[dest], gpr_load_U, reg_offs,
				true, !dont_upd_U).add_comment(
  	comment.str()));
}

void bgp_wilson_t::upd_load_reg_psi(unsigned fw, unsigned mu, int a)
{
  unsigned i = mu + 4 * fw;
  bool upd = false;
  unsigned offs = 2*i;
  if (fw == 0 && mu == 0)
  {
    upd = true;
    offs = nb_array_offs;
  }
  bool shift = 1 - mu % 2;
  unsigned sh = (1 - mu % 2) * 16 + 6; // PREC
  if (a == 0)
  {
    if (mu % 2 == 0)
      add(bgp_code_element_t::load(gpr_nb_idx, gpr_nb, offs, upd));
    return;
  }
  if (a == 1)
  {
    cr.get_this_reg(cr_0);
    gpr_scratch_nb = gpr.get_reg();
    add(bgp_code_element_t::and_i(
	  gpr_scratch_nb, gpr_nb_idx, 1 << NB_ARRAY_RECV_BIT, shift, true));
  }
  if (a == 2)
  {
    add(bgp_code_element_t::rlwinm(gpr_scratch_nb, gpr_nb_idx, sh, 11, 25)); // PREC
    return;
  }
  if (a == 3)
  {
    add(bgp_code_element_t::select(gpr_load_psi[mu], gpr_psi_base,
				   gpr_psi_nb_buffer, bgp_code_element_t::cr_field_eq));
    cr.release_reg(cr_0);
    return;
  }
  if (a == 4)
  {
    add(bgp_code_element_t::add(gpr_load_psi[mu], gpr_load_psi[mu], gpr_scratch_nb));
    gpr.release_reg(gpr_scratch_nb);
    return;
  }
  if (a == 5)
  {
    if (fw == 0 && mu == 3)
      add(bgp_code_element_t::prefetch(gpr_48, gpr_load_psi[mu]));
    if (fw == 0 && mu == 2)
      add(bgp_code_element_t::prefetch(gpr_16, gpr_load_psi[mu]));
    if (fw == 1 && mu == 3)
      add(bgp_code_element_t::prefetch(gpr_16, gpr_load_psi[mu]));
    if (fw == 1 && mu == 2)
      add(bgp_code_element_t::prefetch(gpr_16, gpr_load_psi[mu]));
    return;
  }
}

unsigned bgp_wilson_t::fpr_psi_idx(unsigned fw, unsigned mu, unsigned col, unsigned hsp, unsigned pass)
{
  const unsigned rot[8] = {0, 3, 1, 2, 5, 6, 4, 7};
  const unsigned offs[12] = {0, 6, 2, 4, 3, 1, 5, 7, 6, 4, 0, 2};

  unsigned i = pass + 2 * (hsp + 2 * col);
  unsigned fwmu = mu + 4 * fw;
  return rot[(fwmu + offs[i]) % 8];
}

// i = 0..11 (even if mu==3)
void bgp_wilson_t::load_psi(unsigned fw, unsigned mu, unsigned i)
{
  unsigned col        = i/4;                 // color
  unsigned pass       = i%2;
  unsigned hsp        = (i%4)/2;             // half-spinor

  unsigned fwmu       = 4 * fw + mu;
  unsigned target_reg = fpr_psi_idx(fw, mu, col, hsp, pass);

  if (mu == 3 && pass == 1)
  {					       
    if (i == 5 || i == 9)
      add(bgp_code_element_t::add_i(gpr_load_psi[mu], gpr_load_psi[mu], -64));
    if (i == 1 || i == 7)
      add(bgp_code_element_t::add_i(gpr_load_psi[mu], gpr_load_psi[mu], 48));
    if (i == 3)
      add(bgp_code_element_t::add_i(gpr_load_psi[mu], gpr_load_psi[mu], 96));
    return;
  }
  unsigned reg_offs= -1;
  if (mu == 3)
  {
    if (i == 0)
      reg_offs = (fw==0 && !dagger_ || fw==1 && dagger_) ? gpr_48 : gpr_16;
    if (i == 2)
      reg_offs = gpr_16; // -48
    if (i == 4)
      reg_offs = gpr_16; //  64
    if (i == 6)
      reg_offs = gpr_16; // -48
    if (i == 8)
      reg_offs = gpr_16; // 112
    if (i == 10)
      reg_offs = gpr_16; //  64
  }
  else
    if (hsp_col[mu][0] == 0)
    {
      if (i % 2 == 1 || i == 4 || i == 10)
	reg_offs = gpr_32;
      if (i == 0)
	reg_offs = gpr_16;
      if (i == 2 || i == 6)
	reg_offs = gpr_m80;
      if (i == 8)
	reg_offs = gpr_80;
    }
    else
    {
      if (i % 4 == 1 || i == 10)
	reg_offs = gpr_48;
      if (i % 4 == 3 || i == 0 || i == 4)
	reg_offs = gpr_16;
      if (i == 2 || i == 6)
	reg_offs = gpr_m96;
      if (i == 8)
	reg_offs = gpr_96;
    }

  ostringstream comment;
  comment << "load psi " << i << " fw " << fw << "mu " << mu;
  add(bgp_code_element_t::fload(fpr_psi_pr[target_reg], gpr_load_psi[mu], reg_offs, true, 
  				!dont_upd_psi).add_comment(comment.str()));
}

// i=0..11
void bgp_wilson_t::pref_psi(unsigned i)
{
  switch(pref_psi_kind)
  {
  case pref_no:
    return;
  case pref_pref:
    if (i % 4 == 0)
      add(bgp_code_element_t::prefetch(gpr_0, gpr_pref_psi));
    if (i % 4 == 2)
      add(bgp_code_element_t::prefetch(gpr_32, gpr_pref_psi));
    if (i % 4 == 3)
      add(bgp_code_element_t::add_i(gpr_pref_psi, gpr_pref_psi, 64)); //PREC
    return;
  case pref_load:
    if (i % 2 == 0)
      add(bgp_code_element_t::load(
	    gpr_pref_psi_d[(i/2) % 3], gpr_pref_psi, 32, true));
  }
}


void bgp_wilson_t::psi_pr(unsigned fw, unsigned mu, unsigned i)
{
  unsigned hsp = i % 2;
  unsigned col = i / 2;
  unsigned dest = fpr_psi_pr[fpr_psi_idx(fw, mu, col, hsp, 0)];
  unsigned src  = fpr_psi_pr[fpr_psi_idx(fw, mu, col, hsp, 1)];
  if (mu != 3)
  {
    if (dagger_)
      fw = 1-fw;
    if ((fw==0 && (hsp_isign[mu][hsp_col[mu][i%2]]%3 == 0))  ||
	(fw==1 && (hsp_isign[mu][hsp_col[mu][i%2]]%3 != 0)))
      add(bgp_code_element_t::fadd_isign(dest, src, hsp_isign[mu][hsp_col[mu][i%2]], fpr_i));
    else
      add(bgp_code_element_t::fadd_isign(dest, src, 3-hsp_isign[mu][hsp_col[mu][i%2]], fpr_i));
  }
}


void bgp_wilson_t::mult(unsigned fw, unsigned mu, unsigned i)
{
  unsigned hspin = i%2;      // half spinor to be updated
  unsigned row   = (i/2)%3;  // row of U
  unsigned pass  = (i/6)%2;  // 2 mult. per component
  unsigned col   = i/12;     // column of U

  bool sum       = (col>0) || (pass>0) || (mu==3); // add to result or initialize?
  bool flip      = false;
  ostringstream comment;
  comment << "mult " << i;

  if (!dagger_ && fw==1 && mu==3 && i>=30)
    flip = true;

  unsigned reg_U    = fpr_U[fpr_U_idx(fw, mu, row + 3 * col)];
  unsigned reg_psi  = fpr_psi_pr[fpr_psi_idx(fw, mu, col, hspin, 0)];
  unsigned reg_hres = fpr_hres[hspin + 2 * row ];
  unsigned reg_res  = fpr_res[hspin + 4 * row + 2];
  bool negate =  (!dagger_ && fw==0 && mu==1 && col==2 && hspin==0) ||
    (dagger_ && fw==0 && mu==1 && col==2 && hspin==1);

  if (!flip)
    add(bgp_code_element_t::cplxmul(reg_hres, reg_U, reg_psi, pass, sum,
				    false, negate).add_comment(comment.str()));
  else
    add(bgp_code_element_t::cplxmul(reg_res, reg_U, reg_psi, pass, sum,
				    false, negate, reg_hres).add_comment(comment.str()));
}


void bgp_wilson_t::add_to_res(unsigned fw, unsigned mu, unsigned i)
{
  ostringstream comment;
  comment << "add to res fw/mu " << fw << "/" << mu << "   i " << i;
  unsigned col    = i/4;
  unsigned hspin  = (i%4) % 2;
  unsigned pass   = (i%4) / 2;
  if (fw == 0 && mu == 0)
    return;
  if (fw == 0 && mu == 1)
    pass = 1 - pass;
  if (mu == 3)   // mu==3 -> projection trivial
    return;
  if (fw==0 && mu==1 && pass==1)
  {
    if (!dagger_)
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col + 2 + hspin], fpr_res[4*col + 1 - hspin],
	    1, fpr_i, fpr_hres[2*col + hsp_col[mu][hspin]]).add_comment(comment.str()));
    else
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col + 2 + hspin], fpr_res[4*col + 1 - hspin],
	    2, fpr_i, fpr_hres[2*col + hsp_col[mu][hspin]]).add_comment(comment.str()));
    return;
  }
  if (pass == 0)
  {
    if (fw == 0 && mu==1 && ((hspin==0 && !dagger_) || (hspin==1 && dagger_)))
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col+hspin], fpr_hres[2*col+hspin], 3, fpr_i).add_comment(comment.str()));
    else
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col+hspin], fpr_hres[2*col+hspin], 0, fpr_i).add_comment(comment.str()));
  }
  else
  {
    if (dagger_)
      fw = 1 - fw;
    if (fw == 0)
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col+2+hspin], fpr_hres[2*col+hsp_col[mu][hspin]],
	    hsp_isign[mu][hspin], fpr_i).add_comment(comment.str()));
    else
      add(bgp_code_element_t::fadd_isign(
	    fpr_res[4*col+2+hspin], fpr_hres[2*col+hsp_col[mu][hspin]],
	    3-hsp_isign[mu][hspin], fpr_i).add_comment(comment.str()));
  }
}

unsigned bgp_wilson_t::store_res_offs_reg(unsigned i)
{
  unsigned reg_offs;
  if (i % 2)
    reg_offs = gpr_16;
  if (i == 0)
    reg_offs = dagger_? gpr_16 : gpr_80;
  if (i == 2 || i == 4 || i == 8 || i == 10)
    reg_offs = gpr_48;
  if (i == 6)
    reg_offs = gpr_16;
  return reg_offs;
}

void bgp_wilson_t::store_res(unsigned i)
{
  unsigned i0 = i;
  ostringstream comment;
  comment << "store " << i;
  unsigned reg_offs = store_res_offs_reg(i);
  if (!dagger_ || i>5)
    i = 4*((i%6) / 2) + 2 + i%2;
  else
    i = 4*((i%6) / 2) + i%2;
  comment << "      (" << i << ")";
  if (i0 == 6)
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, dagger_? -128 : -192));
  add(bgp_code_element_t::fstore(fpr_res[i], gpr_store_r, reg_offs, true, !dont_upd_r).add_comment(
  	comment.str()));
}

void bgp_wilson_t::upd_shadow_ctr(unsigned ctr, unsigned i)
{
  if (i == 0)
  {
    gpr_scratch_sctr = gpr.get_reg();
    add(bgp_code_element_t::load(gpr_scratch_sctr, gpr_nb, 2 * nb_array_offs + 4*ctr));
  }
  if (i == 1)
    add(bgp_code_element_t::rlwinm(gpr_scratch_sctr, gpr_scratch_sctr, 18, 14, 30));
  if (i == 2)
    add(bgp_code_element_t::rlwinm(gpr_scratch_sctr, gpr_scratch_sctr,  0, 30, 14));
  if (i == 3)
    add(bgp_code_element_t::add(gpr_counter[ctr], gpr_counter[ctr], gpr_scratch_sctr));
  if (i == 4)
    add(bgp_code_element_t::rlwinm(gpr_scratch_sctr, gpr_scratch_sctr,  31, 0, 31));
  if (i == 5)
  {
    add(bgp_code_element_t::add(gpr_counter[ctr], gpr_counter[ctr], gpr_scratch_sctr));
    gpr.release_reg(gpr_scratch_sctr);
  }

}

void bgp_wilson_t::upd_hw_counters(unsigned fw, unsigned mu, unsigned i)
{
  unsigned fwmu = mu + 4 * fw;
  bool high = fwmu % 2 == 0;
  unsigned idx = fwmu / 2;
  if (i == 0)
  {
    gpr_scratch_sctr = gpr.get_reg();
    unsigned offs = fwmu * sizeof(DMA_CounterHw_t) + offsetof(DMA_CounterHw_t, counter);
    add(bgp_code_element_t::load(gpr_scratch_sctr, gpr_counter_base, offs));
  }
  if (i == 1)
  {
    unsigned sh = high? 10 : 26; //PREC
    unsigned mb = high? 0 : 16;
    add(bgp_code_element_t::rlwimi(gpr_counter_hw[idx], gpr_scratch_sctr, sh, mb, mb+15));
    gpr.release_reg(gpr_scratch_sctr);
  }
}

void bgp_wilson_t::cmp_counters(unsigned i)
{
  if (i == 0)
  {
    gpr.get_regs(2, gpr_scratch_ctr);
    add(bgp_code_element_t::add(gpr_scratch_ctr[0], gpr_counter[0], gpr_counter_hw[0]));
  }
  if (i == 1)
    add(bgp_code_element_t::add(gpr_scratch_ctr[1], gpr_counter[1], gpr_counter_hw[1]));
  if (i == 2)
    add(bgp_code_element_t::andd(gpr_scratch_ctr[0], gpr_scratch_ctr[0], gpr_scratch_ctr[1]));
  if (i == 3)
    add(bgp_code_element_t::add(gpr_scratch_ctr[1], gpr_counter[2], gpr_counter_hw[2]));
  if (i == 4)
    add(bgp_code_element_t::andd(gpr_scratch_ctr[0], gpr_scratch_ctr[0], gpr_scratch_ctr[1]));
  if (i == 5)
    add(bgp_code_element_t::add(gpr_scratch_ctr[1], gpr_counter[3], gpr_counter_hw[3]));
  if (i == 6)
    add(bgp_code_element_t::andd(gpr_scratch_ctr[0], gpr_scratch_ctr[0], gpr_scratch_ctr[1]));
  if (i == 7)
    add(bgp_code_element_t::rlwinm(gpr_scratch_ctr[1], gpr_scratch_ctr[0], 16, 0, 0));
  if (i == 8)
  {
    cr.get_this_reg(cr_0);
    add(bgp_code_element_t::andd(gpr_scratch_ctr[0], gpr_scratch_ctr[0], gpr_scratch_ctr[1], true));
    gpr.release_regs(gpr_scratch_ctr);
  }
}


void bgp_wilson_t::prolog()
{
  add(bgp_code_element_t::comment("start prolog"));
  for (unsigned fw=0; fw<1; ++fw)
    for (unsigned mu=0; mu<4; ++mu)
      for (unsigned a=0; a<6; ++a)
	upd_load_reg_psi(fw, mu, a);
  add(bgp_code_element_t::jump(wait_loop_label, true));

  load_psi(0, 0, 0);
  load_psi(0, 0, 1);
  load_psi(0, 0, 4);
  load_psi(0, 0, 5);
  load_psi(0, 0, 2);
  load_psi(0, 0, 3);

  psi_pr(0, 0, 0);
  psi_pr(0, 0, 1);

  load_psi(0, 0, 8);

  for (unsigned i=0; i<5; ++i)
    load_U(0, 0, i);
}

void bgp_wilson_t::epilog()
{
  for (unsigned i=6; i<12; ++i)
    store_res(i);
  unsigned reg_scratch  = gpr.get_reg();
  unsigned reg_scratch2 = gpr.get_reg();
  if (T0_LAST)
  {
    add(bgp_code_element_t::add_i(reg_scratch, gpr_sp, st_offs_res0 + (dagger_? 192-16 : 192-48)));//PREC
    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fload(fpr_res[i], reg_scratch, store_res_offs_reg(i), true, true));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_res_addr));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, dagger_? -16 : -48));//PREC
    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fstore(fpr_res[i], gpr_store_r, store_res_offs_reg(i), true, true));



    add(bgp_code_element_t::add_i(reg_scratch, gpr_sp, st_offs_res0 + (dagger_? 16 : -80)));//PREC
    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fload(fpr_res[i], reg_scratch, store_res_offs_reg(i), true, true));

    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_res_addr));
    add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, dagger_? 16-192 : -80-192));//PREC

    add(bgp_code_element_t::load(reg_scratch2, gpr_sp, st_offs_vol));
//    add(bgp_code_element_t::add_i(reg_scratch2, reg_scratch2, -1));//PREC
    add(bgp_code_element_t::mul_i(reg_scratch2, reg_scratch2, 192));//PREC
    add(bgp_code_element_t::add(gpr_store_r, gpr_store_r, reg_scratch2));//PREC

    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fstore(fpr_res[i], gpr_store_r, store_res_offs_reg(i), true, true));

  }
  else
  {
    if (!dagger_)
      add(bgp_code_element_t::add_i(reg_scratch, gpr_sp, st_offs_res0+192-48));//PREC
    else
      add(bgp_code_element_t::add_i(reg_scratch, gpr_sp, st_offs_res0+192-16));//PREC
    
    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fload(fpr_res[i], reg_scratch, store_res_offs_reg(i), true, true));
    add(bgp_code_element_t::load(gpr_store_r, gpr_sp, st_offs_res_addr));
    if (!dagger_)
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, -48));//PREC
    else
      add(bgp_code_element_t::add_i(gpr_store_r, gpr_store_r, -16));//PREC
    for (unsigned i=0; i<6; ++i)
      add(bgp_code_element_t::fstore(fpr_res[i], gpr_store_r, store_res_offs_reg(i), true, true));
  }
}

void bgp_wilson_t::wait_counters()
{
  add(label.set_label(wait_loop_label));
  string loop_label("");
  add(label.set_label(loop_label));
  for (unsigned fw=0; fw<2; ++fw)
    for (unsigned mu=0; mu<4; ++mu)
      for (unsigned i=0; i<2; ++i)
  	upd_hw_counters(fw, mu, i);

  vector<unsigned> reg_scratch;
  gpr.get_regs(2, reg_scratch);
  add(bgp_code_element_t::add(reg_scratch[0], gpr_counter[0], gpr_counter_hw[0]));
  add(bgp_code_element_t::add(reg_scratch[1], gpr_counter[1], gpr_counter_hw[1]));
  add(bgp_code_element_t::andd(reg_scratch[0], reg_scratch[0], reg_scratch[1]));

  add(bgp_code_element_t::add(reg_scratch[1], gpr_counter[2], gpr_counter_hw[2]));
  add(bgp_code_element_t::andd(reg_scratch[0], reg_scratch[0], reg_scratch[1]));
  add(bgp_code_element_t::add(reg_scratch[1], gpr_counter[3], gpr_counter_hw[3]));
  add(bgp_code_element_t::andd(reg_scratch[0], reg_scratch[0], reg_scratch[1]));
  add(bgp_code_element_t::rlwinm(reg_scratch[1], reg_scratch[0], 16, 0, 0));
  add(bgp_code_element_t::andd(reg_scratch[0], reg_scratch[0], reg_scratch[1], true));

  add(bgp_code_element_t::jump_cond(loop_label, cr_0, bgp_code_element_t::cr_field_lt, 0));
  add(bgp_code_element_t::mbar());
  add(bgp_code_element_t::jump_lr());
  gpr.release_regs(reg_scratch);
}

void bgp_wilson_t::mult_Upsi(unsigned fw, unsigned mu)
{
  unsigned fw_n  = fw;
  unsigned mu_n  = (mu+1) % 4;
  if (mu_n == 0)
    fw_n = 1 - fw_n;
  unsigned fw_nn = fw_n;
  unsigned mu_nn = (mu_n+1) % 4;
  if (mu_nn == 0)
    fw_nn = 1-fw_nn;

  vector<unsigned> save_fpr_hres(0);

// on first direction, store directly into res
  if (fw == 0 && mu == 0)
  {
    save_fpr_hres = fpr_hres;
    for (unsigned i=0; i<6; ++i)
      fpr_hres[i] = fpr_res[4*(i/2)+(i%2)];
  }
// in time direction, accumulate directly (projection trivial)
  if (mu == 3)
  {
    save_fpr_hres = fpr_hres;
    for (int i=0; i<6; i++)
      if (dagger_)
	fpr_hres[i] = fpr_res[4*(i/2) + (i%2)+2*fw];
      else
	fpr_hres[i] = fpr_res[4*(i/2) + (i%2)+2*(1-fw)];
  }

  mult(fw, mu,  0);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 0);
  mult(fw, mu,  1);
  load_psi(fw, mu, 9);
  mult(fw, mu,  2);
  mult(fw, mu,  3);
  load_psi(fw, mu, 6);
  mult(fw, mu,  4);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 0);
  mult(fw, mu,  5);
  load_psi(fw, mu, 7);
  mult(fw, mu,  6);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 1);
  mult(fw, mu,  7);
  load_U(fw, mu, 5);
  mult(fw, mu,  8);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 2);
  psi_pr(fw, mu, 2);
  psi_pr(fw, mu, 3);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 3);
  mult(fw, mu,  9);
  load_U(fw, mu, 6);
  mult(fw, mu, 10);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 4);
  mult(fw, mu, 11);
  load_U(fw, mu, 7);
  load_psi(fw_n, mu_n, 0);
  mult(fw, mu, 12);
  if ((fw == 1 && mu==3) || (fw == 0 && mu==0))
    store_res(6*(1-fw) + 5);
  mult(fw, mu, 13);
  load_psi(fw_n, mu_n, 1);
  mult(fw, mu, 14);
  mult(fw, mu, 15);
  load_psi(fw, mu, 10);
  mult(fw, mu, 16);
  if (mu+4*fw < 6)
    pref_psi(2 * (mu+4*fw));
  mult(fw, mu, 17);
  load_psi(fw, mu, 11);
  mult(fw, mu, 18);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 1);
  mult(fw, mu, 19);
  load_U(fw, mu, 8);
  mult(fw, mu, 20);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 2);
  psi_pr(fw, mu, 4);
  psi_pr(fw, mu, 5);
  mult(fw, mu, 21);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 3);
  load_U(fw_n, mu_n, 0);
  mult(fw, mu, 22);
  mult(fw, mu, 23);
  load_U(fw_n, mu_n, 1);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 4);
  load_psi(fw_n, mu_n, 4);
  mult(fw, mu, 24);
  mult(fw, mu, 25);
  load_psi(fw_n, mu_n, 5);
  mult(fw, mu, 26);
  if (!dont_upd_psi)
    upd_load_reg_psi(1-fw, mu, 5);
  mult(fw, mu, 27);
  load_psi(fw_n, mu_n, 2);
  mult(fw, mu, 28);
  mult(fw, mu, 29);
  load_psi(fw_n, mu_n, 3);
  mult(fw, mu, 30);
  if (mu+4*fw < 6)
    pref_psi(2 * (mu+4*fw) + 1);
  mult(fw, mu, 31);
  load_U(fw_n, mu_n, 2);
  mult(fw, mu, 32);
  psi_pr(fw_n, mu_n, 0);
  psi_pr(fw_n, mu_n, 1);
  mult(fw, mu, 33);
  load_U(fw_n, mu_n, 3);
  mult(fw, mu, 34);
  mult(fw, mu, 35);
  load_U(fw_n, mu_n, 4);
  load_psi(fw_n, mu_n, 8);


  add_to_res(fw, mu, 0);
  add_to_res(fw, mu, 1);
  add_to_res(fw, mu, 2);
  add_to_res(fw, mu, 3);
  add_to_res(fw, mu, 4);
  add_to_res(fw, mu, 5);
  add_to_res(fw, mu, 6);
  add_to_res(fw, mu, 7);
  add_to_res(fw, mu, 8);
  add_to_res(fw, mu, 9);
  add_to_res(fw, mu,10);
  add_to_res(fw, mu,11);

  if (fw == 0)
  {
    upd_shadow_ctr(mu, 0);
    upd_shadow_ctr(mu, 1);
    upd_shadow_ctr(mu, 2);
    upd_shadow_ctr(mu, 3);
    upd_shadow_ctr(mu, 4);
    upd_shadow_ctr(mu, 5);
  }

//restore register allocation
  if (save_fpr_hres.size())    
    fpr_hres = save_fpr_hres;
}

void bgp_wilson_t::debug_regs()
{
  unsigned st_sz = 32;

  add(asm_code_element_t(".globl\tdebug_regs"));
  add(asm_code_element_t("debug_regs:"));

  add(bgp_code_element_t::store(gpr_sp, gpr_sp, -int(st_sz), true));
  add(bgp_code_element_t::store(gpr_0, gpr_sp, 4));
  add(bgp_code_element_t::mflr(gpr_0));
  add(bgp_code_element_t::store(gpr_0, gpr_sp, 8));
  add(bgp_code_element_t::store(3, gpr_sp, 12));
  add(bgp_code_element_t::store(4, gpr_sp, 16));

  add(bgp_code_element_t::load_hi(3, 0, "debug_buffer_asm"));
  add(bgp_code_element_t::load_lo(3, 3, "debug_buffer_asm"));
  add(bgp_code_element_t::load(3, 3, 0));
  add(bgp_code_element_t::add_i(3, 3, -4));
  add(bgp_code_element_t::load_i(4, 1));
  add(bgp_code_element_t::store(4, 3, 4, true));
  for (unsigned i=0; i<32; ++i)
    if (i == 0)
    {
      add(bgp_code_element_t::load(4, gpr_sp, 4));
      add(bgp_code_element_t::store(4, 3, 4, true));
    }
    else if (i == 3)
    {
      add(bgp_code_element_t::load(4, gpr_sp, 12));
      add(bgp_code_element_t::store(4, 3, 4, true));
    }
    else if (i == 4)
    {
      add(bgp_code_element_t::load(4, gpr_sp, 16));
      add(bgp_code_element_t::store(4, 3, 4, true));
    }
    else
      add(bgp_code_element_t::store(i, 3, 4, true));

  add(bgp_code_element_t::mfctr(4));
  add(bgp_code_element_t::store(4, 3, 4, true));
  add(bgp_code_element_t::mfcr(4));
  add(bgp_code_element_t::store(4, 3, 4, true));
	
  add(bgp_code_element_t::add_i(3, 3, -12));
  add(bgp_code_element_t::load_i(4, 16));

  for (unsigned i=0; i<32; ++i)
    add(bgp_code_element_t::fstore(i, 3, 4, true, true));
    
  add(bgp_code_element_t::add_i(3, 3, 16));
      
  add(bgp_code_element_t::load_hi(4, 0, "debug_buffer_asm"));
  add(bgp_code_element_t::load_lo(4, 4, "debug_buffer_asm"));
  add(bgp_code_element_t::store(3, 4, 0));
  

  add(bgp_code_element_t::load(4, gpr_sp, 16));
  add(bgp_code_element_t::load(3, gpr_sp, 12));
  add(bgp_code_element_t::load(gpr_0, gpr_sp, 8));
  add(bgp_code_element_t::mtlr(gpr_0));
  add(bgp_code_element_t::load(gpr_0, gpr_sp, 4));


  add(bgp_code_element_t::add_i(gpr_sp, gpr_sp, st_sz));
  add(asm_code_element_t("        bclr     20, 0"));
}

const unsigned bgp_wilson_t::gpr_0(0);
const unsigned bgp_wilson_t::gpr_16(0);
const unsigned bgp_wilson_t::gpr_sp(1);
const unsigned bgp_wilson_t::gpr_psi_base(3);
const unsigned bgp_wilson_t::gpr_store_r(4);
const unsigned bgp_wilson_t::gpr_data(5);

const unsigned bgp_wilson_t::cr_0(0);


// 0 : 1, 1 : i, 2 : -i, 3 : -1
const unsigned bgp_wilson_t::hsp_isign[3][2] = {{1, 1}, {0, 3}, {1, 2}};
const unsigned bgp_wilson_t::hsp_col[3][2]   = {{1, 0}, {1, 0}, {0, 1}};
