/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFI_ATHLON;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#include <stdio.h>
#include <string.h>
#include "common.h"

#ifdef BUILD_KERNEL
#include "kernel_ATHLON.h"
#endif

#undef DEBUG

static void init_parameter(void);

gotoblas_t TABLE_NAME = {
  DTB_DEFAULT_ENTRIES ,

  GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN,

#ifdef BUILD_BFLOAT16
  0, 0, 0,
  SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N,
#ifdef SBGEMM_DEFAULT_UNROLL_MN
 SBGEMM_DEFAULT_UNROLL_MN,
#else
 MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N),
#endif

  sbstobf16_k_ATHLON, sbdtobf16_k_ATHLON, sbf16tos_k_ATHLON, dbf16tod_k_ATHLON,

  samax_k_ATHLON,  samin_k_ATHLON,  smax_k_ATHLON,  smin_k_ATHLON,
  isamax_k_ATHLON, isamin_k_ATHLON, ismax_k_ATHLON, ismin_k_ATHLON,
  snrm2_k_ATHLON,  sasum_k_ATHLON, ssum_k_ATHLON, scopy_k_ATHLON, sbdot_k_ATHLON,
  dsdot_k_ATHLON,
  srot_k_ATHLON,   saxpy_k_ATHLON,  sscal_k_ATHLON, sswap_k_ATHLON,
  sbgemv_n_ATHLON, sbgemv_t_ATHLON, sger_k_ATHLON,
  ssymv_L_ATHLON, ssymv_U_ATHLON,

  sbgemm_kernel_ATHLON, sbgemm_beta_ATHLON,
#if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N
  sbgemm_incopy_ATHLON, sbgemm_itcopy_ATHLON,
#else
  sbgemm_oncopy_ATHLON, sbgemm_otcopy_ATHLON,
#endif
  sbgemm_oncopy_ATHLON, sbgemm_otcopy_ATHLON,

  strsm_kernel_LN_ATHLON, strsm_kernel_LT_ATHLON, strsm_kernel_RN_ATHLON, strsm_kernel_RT_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  strsm_iunucopy_ATHLON, strsm_iunncopy_ATHLON, strsm_iutucopy_ATHLON, strsm_iutncopy_ATHLON,
  strsm_ilnucopy_ATHLON, strsm_ilnncopy_ATHLON, strsm_iltucopy_ATHLON, strsm_iltncopy_ATHLON,
#else
  strsm_ounucopy_ATHLON, strsm_ounncopy_ATHLON, strsm_outucopy_ATHLON, strsm_outncopy_ATHLON,
  strsm_olnucopy_ATHLON, strsm_olnncopy_ATHLON, strsm_oltucopy_ATHLON, strsm_oltncopy_ATHLON,
#endif
  strsm_ounucopy_ATHLON, strsm_ounncopy_ATHLON, strsm_outucopy_ATHLON, strsm_outncopy_ATHLON,
  strsm_olnucopy_ATHLON, strsm_olnncopy_ATHLON, strsm_oltucopy_ATHLON, strsm_oltncopy_ATHLON,
  strmm_kernel_RN_ATHLON, strmm_kernel_RT_ATHLON, strmm_kernel_LN_ATHLON, strmm_kernel_LT_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  strmm_iunucopy_ATHLON, strmm_iunncopy_ATHLON, strmm_iutucopy_ATHLON, strmm_iutncopy_ATHLON,
  strmm_ilnucopy_ATHLON, strmm_ilnncopy_ATHLON, strmm_iltucopy_ATHLON, strmm_iltncopy_ATHLON,
#else
  strmm_ounucopy_ATHLON, strmm_ounncopy_ATHLON, strmm_outucopy_ATHLON, strmm_outncopy_ATHLON,
  strmm_olnucopy_ATHLON, strmm_olnncopy_ATHLON, strmm_oltucopy_ATHLON, strmm_oltncopy_ATHLON,
#endif
  strmm_ounucopy_ATHLON, strmm_ounncopy_ATHLON, strmm_outucopy_ATHLON, strmm_outncopy_ATHLON,
  strmm_olnucopy_ATHLON, strmm_olnncopy_ATHLON, strmm_oltucopy_ATHLON, strmm_oltncopy_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  ssymm_iutcopy_ATHLON, ssymm_iltcopy_ATHLON,
#else
  ssymm_outcopy_ATHLON, ssymm_oltcopy_ATHLON,
#endif
  ssymm_outcopy_ATHLON, ssymm_oltcopy_ATHLON,

#ifndef NO_LAPACK
  sneg_tcopy_ATHLON, slaswp_ncopy_ATHLON,
#else
  NULL,NULL,
#endif
#ifdef SMALL_MATRIX_OPT
  sbgemm_small_matrix_permit_ATHLON,
  sbgemm_small_kernel_nn_ATHLON, sbgemm_small_kernel_nt_ATHLON, sbgemm_small_kernel_tn_ATHLON, sbgemm_small_kernel_tt_ATHLON,
  sbgemm_small_kernel_b0_nn_ATHLON, sbgemm_small_kernel_b0_nt_ATHLON, sbgemm_small_kernel_b0_tn_ATHLON, sbgemm_small_kernel_b0_tt_ATHLON,
#endif
#endif

#if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1)
  0, 0, 0,
  SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N,
#ifdef SGEMM_DEFAULT_UNROLL_MN
 SGEMM_DEFAULT_UNROLL_MN,
#else
 MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),
#endif
#endif

#ifdef HAVE_EXCLUSIVE_CACHE
  1,
#else
  0,
#endif

#if (BUILD_SINGLE==1 ) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1)
  samax_k_ATHLON,  samin_k_ATHLON,  smax_k_ATHLON,  smin_k_ATHLON,
  isamax_k_ATHLON, isamin_k_ATHLON, ismax_k_ATHLON, ismin_k_ATHLON,
  snrm2_k_ATHLON,  sasum_k_ATHLON,
#endif 
#if BUILD_SINGLE == 1  
  ssum_k_ATHLON,
#endif

#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1)
  scopy_k_ATHLON, sdot_k_ATHLON,
//  dsdot_k_ATHLON,
  srot_k_ATHLON,   saxpy_k_ATHLON,  
#endif
#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1)
  sscal_k_ATHLON,
#endif 
#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1)
  sswap_k_ATHLON,
  sgemv_n_ATHLON,  sgemv_t_ATHLON,
#endif
#if BUILD_SINGLE == 1  
  sger_k_ATHLON,
  ssymv_L_ATHLON, ssymv_U_ATHLON,
#endif

#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1)
#ifdef ARCH_X86_64
  sgemm_direct_ATHLON,
  sgemm_direct_performant_ATHLON,	
#endif

  sgemm_kernel_ATHLON, sgemm_beta_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  sgemm_incopy_ATHLON, sgemm_itcopy_ATHLON,
#else
  sgemm_oncopy_ATHLON, sgemm_otcopy_ATHLON,
#endif
  sgemm_oncopy_ATHLON, sgemm_otcopy_ATHLON,
#endif

#if BUILD_SINGLE == 1
#ifdef SMALL_MATRIX_OPT
  sgemm_small_matrix_permit_ATHLON,
  sgemm_small_kernel_nn_ATHLON, sgemm_small_kernel_nt_ATHLON, sgemm_small_kernel_tn_ATHLON, sgemm_small_kernel_tt_ATHLON,
  sgemm_small_kernel_b0_nn_ATHLON, sgemm_small_kernel_b0_nt_ATHLON, sgemm_small_kernel_b0_tn_ATHLON, sgemm_small_kernel_b0_tt_ATHLON,
#endif
#endif

#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) 
  strsm_kernel_LN_ATHLON, strsm_kernel_LT_ATHLON, strsm_kernel_RN_ATHLON, strsm_kernel_RT_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  strsm_iunucopy_ATHLON, strsm_iunncopy_ATHLON, strsm_iutucopy_ATHLON, strsm_iutncopy_ATHLON,
  strsm_ilnucopy_ATHLON, strsm_ilnncopy_ATHLON, strsm_iltucopy_ATHLON, strsm_iltncopy_ATHLON,
#else
  strsm_ounucopy_ATHLON, strsm_ounncopy_ATHLON, strsm_outucopy_ATHLON, strsm_outncopy_ATHLON,
  strsm_olnucopy_ATHLON, strsm_olnncopy_ATHLON, strsm_oltucopy_ATHLON, strsm_oltncopy_ATHLON,
#endif
  strsm_ounucopy_ATHLON, strsm_ounncopy_ATHLON, strsm_outucopy_ATHLON, strsm_outncopy_ATHLON,
  strsm_olnucopy_ATHLON, strsm_olnncopy_ATHLON, strsm_oltucopy_ATHLON, strsm_oltncopy_ATHLON,
#endif
#if BUILD_SINGLE == 1
  strmm_kernel_RN_ATHLON, strmm_kernel_RT_ATHLON, strmm_kernel_LN_ATHLON, strmm_kernel_LT_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  strmm_iunucopy_ATHLON, strmm_iunncopy_ATHLON, strmm_iutucopy_ATHLON, strmm_iutncopy_ATHLON,
  strmm_ilnucopy_ATHLON, strmm_ilnncopy_ATHLON, strmm_iltucopy_ATHLON, strmm_iltncopy_ATHLON,
#else
  strmm_ounucopy_ATHLON, strmm_ounncopy_ATHLON, strmm_outucopy_ATHLON, strmm_outncopy_ATHLON,
  strmm_olnucopy_ATHLON, strmm_olnncopy_ATHLON, strmm_oltucopy_ATHLON, strmm_oltncopy_ATHLON,
#endif
  strmm_ounucopy_ATHLON, strmm_ounncopy_ATHLON, strmm_outucopy_ATHLON, strmm_outncopy_ATHLON,
  strmm_olnucopy_ATHLON, strmm_olnncopy_ATHLON, strmm_oltucopy_ATHLON, strmm_oltncopy_ATHLON,
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
  ssymm_iutcopy_ATHLON, ssymm_iltcopy_ATHLON,
#else
  ssymm_outcopy_ATHLON, ssymm_oltcopy_ATHLON,
#endif
  ssymm_outcopy_ATHLON, ssymm_oltcopy_ATHLON,
#endif
#if  (BUILD_SINGLE==1) || (BUILD_DOUBLE==1)
#ifndef NO_LAPACK
  sneg_tcopy_ATHLON, slaswp_ncopy_ATHLON,
#else
  NULL,NULL,
#endif
#endif

#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)  
  0, 0, 0,
  DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N,
#ifdef DGEMM_DEFAULT_UNROLL_MN
 DGEMM_DEFAULT_UNROLL_MN,
#else
 MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),
#endif
#endif


#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)  
  damax_k_ATHLON,  damin_k_ATHLON,  dmax_k_ATHLON,  dmin_k_ATHLON,
  idamax_k_ATHLON, idamin_k_ATHLON, idmax_k_ATHLON, idmin_k_ATHLON,
  dnrm2_k_ATHLON, dasum_k_ATHLON,
#endif  
#if  (BUILD_DOUBLE==1)  
  dsum_k_ATHLON,
#endif
#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)  
  dcopy_k_ATHLON, ddot_k_ATHLON,
#endif
#if  (BUILD_SINGLE==1) || (BUILD_DOUBLE==1)  
  dsdot_k_ATHLON,
#endif
#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)  
  drot_k_ATHLON,
  daxpy_k_ATHLON,
  dscal_k_ATHLON, 
  dswap_k_ATHLON,
  dgemv_n_ATHLON,  dgemv_t_ATHLON,
#endif
#if  (BUILD_DOUBLE==1)  
  dger_k_ATHLON,
  dsymv_L_ATHLON,  dsymv_U_ATHLON,
#endif

#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)  
  dgemm_kernel_ATHLON, dgemm_beta_ATHLON,
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
  dgemm_incopy_ATHLON, dgemm_itcopy_ATHLON,
#else
  dgemm_oncopy_ATHLON, dgemm_otcopy_ATHLON,
#endif
  dgemm_oncopy_ATHLON, dgemm_otcopy_ATHLON,
#endif

#if  (BUILD_DOUBLE==1)  
#ifdef SMALL_MATRIX_OPT
  dgemm_small_matrix_permit_ATHLON,
  dgemm_small_kernel_nn_ATHLON, dgemm_small_kernel_nt_ATHLON, dgemm_small_kernel_tn_ATHLON, dgemm_small_kernel_tt_ATHLON,
  dgemm_small_kernel_b0_nn_ATHLON, dgemm_small_kernel_b0_nt_ATHLON, dgemm_small_kernel_b0_tn_ATHLON, dgemm_small_kernel_b0_tt_ATHLON,
#endif
  dtrsm_kernel_LN_ATHLON, dtrsm_kernel_LT_ATHLON, dtrsm_kernel_RN_ATHLON, dtrsm_kernel_RT_ATHLON,
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
  dtrsm_iunucopy_ATHLON, dtrsm_iunncopy_ATHLON, dtrsm_iutucopy_ATHLON, dtrsm_iutncopy_ATHLON,
  dtrsm_ilnucopy_ATHLON, dtrsm_ilnncopy_ATHLON, dtrsm_iltucopy_ATHLON, dtrsm_iltncopy_ATHLON,
#else
  dtrsm_ounucopy_ATHLON, dtrsm_ounncopy_ATHLON, dtrsm_outucopy_ATHLON, dtrsm_outncopy_ATHLON,
  dtrsm_olnucopy_ATHLON, dtrsm_olnncopy_ATHLON, dtrsm_oltucopy_ATHLON, dtrsm_oltncopy_ATHLON,
#endif
  dtrsm_ounucopy_ATHLON, dtrsm_ounncopy_ATHLON, dtrsm_outucopy_ATHLON, dtrsm_outncopy_ATHLON,
  dtrsm_olnucopy_ATHLON, dtrsm_olnncopy_ATHLON, dtrsm_oltucopy_ATHLON, dtrsm_oltncopy_ATHLON,
  dtrmm_kernel_RN_ATHLON, dtrmm_kernel_RT_ATHLON, dtrmm_kernel_LN_ATHLON, dtrmm_kernel_LT_ATHLON,
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
  dtrmm_iunucopy_ATHLON, dtrmm_iunncopy_ATHLON, dtrmm_iutucopy_ATHLON, dtrmm_iutncopy_ATHLON,
  dtrmm_ilnucopy_ATHLON, dtrmm_ilnncopy_ATHLON, dtrmm_iltucopy_ATHLON, dtrmm_iltncopy_ATHLON,
#else
  dtrmm_ounucopy_ATHLON, dtrmm_ounncopy_ATHLON, dtrmm_outucopy_ATHLON, dtrmm_outncopy_ATHLON,
  dtrmm_olnucopy_ATHLON, dtrmm_olnncopy_ATHLON, dtrmm_oltucopy_ATHLON, dtrmm_oltncopy_ATHLON,
#endif
  dtrmm_ounucopy_ATHLON, dtrmm_ounncopy_ATHLON, dtrmm_outucopy_ATHLON, dtrmm_outncopy_ATHLON,
  dtrmm_olnucopy_ATHLON, dtrmm_olnncopy_ATHLON, dtrmm_oltucopy_ATHLON, dtrmm_oltncopy_ATHLON,
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
  dsymm_iutcopy_ATHLON, dsymm_iltcopy_ATHLON,
#else
  dsymm_outcopy_ATHLON, dsymm_oltcopy_ATHLON,
#endif
  dsymm_outcopy_ATHLON, dsymm_oltcopy_ATHLON,

#ifndef NO_LAPACK
  dneg_tcopy_ATHLON, dlaswp_ncopy_ATHLON,
#else
  NULL, NULL,
#endif

#endif

#ifdef EXPRECISION

  0, 0, 0,
  QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N),

  qamax_k_ATHLON,  qamin_k_ATHLON,  qmax_k_ATHLON,  qmin_k_ATHLON,
  iqamax_k_ATHLON, iqamin_k_ATHLON, iqmax_k_ATHLON, iqmin_k_ATHLON,
  qnrm2_k_ATHLON,  qasum_k_ATHLON,  qsum_k_ATHLON, qcopy_k_ATHLON, qdot_k_ATHLON,
  qrot_k_ATHLON,   qaxpy_k_ATHLON,  qscal_k_ATHLON, qswap_k_ATHLON,
  qgemv_n_ATHLON,  qgemv_t_ATHLON,  qger_k_ATHLON,
  qsymv_L_ATHLON,  qsymv_U_ATHLON,

  qgemm_kernel_ATHLON, qgemm_beta_ATHLON,
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
  qgemm_incopy_ATHLON, qgemm_itcopy_ATHLON,
#else
  qgemm_oncopy_ATHLON, qgemm_otcopy_ATHLON,
#endif
  qgemm_oncopy_ATHLON, qgemm_otcopy_ATHLON,
  qtrsm_kernel_LN_ATHLON, qtrsm_kernel_LT_ATHLON, qtrsm_kernel_RN_ATHLON, qtrsm_kernel_RT_ATHLON,
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
  qtrsm_iunucopy_ATHLON, qtrsm_iunncopy_ATHLON, qtrsm_iutucopy_ATHLON, qtrsm_iutncopy_ATHLON,
  qtrsm_ilnucopy_ATHLON, qtrsm_ilnncopy_ATHLON, qtrsm_iltucopy_ATHLON, qtrsm_iltncopy_ATHLON,
#else
  qtrsm_ounucopy_ATHLON, qtrsm_ounncopy_ATHLON, qtrsm_outucopy_ATHLON, qtrsm_outncopy_ATHLON,
  qtrsm_olnucopy_ATHLON, qtrsm_olnncopy_ATHLON, qtrsm_oltucopy_ATHLON, qtrsm_oltncopy_ATHLON,
#endif
  qtrsm_ounucopy_ATHLON, qtrsm_ounncopy_ATHLON, qtrsm_outucopy_ATHLON, qtrsm_outncopy_ATHLON,
  qtrsm_olnucopy_ATHLON, qtrsm_olnncopy_ATHLON, qtrsm_oltucopy_ATHLON, qtrsm_oltncopy_ATHLON,
  qtrmm_kernel_RN_ATHLON, qtrmm_kernel_RT_ATHLON, qtrmm_kernel_LN_ATHLON, qtrmm_kernel_LT_ATHLON,
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
  qtrmm_iunucopy_ATHLON, qtrmm_iunncopy_ATHLON, qtrmm_iutucopy_ATHLON, qtrmm_iutncopy_ATHLON,
  qtrmm_ilnucopy_ATHLON, qtrmm_ilnncopy_ATHLON, qtrmm_iltucopy_ATHLON, qtrmm_iltncopy_ATHLON,
#else
  qtrmm_ounucopy_ATHLON, qtrmm_ounncopy_ATHLON, qtrmm_outucopy_ATHLON, qtrmm_outncopy_ATHLON,
  qtrmm_olnucopy_ATHLON, qtrmm_olnncopy_ATHLON, qtrmm_oltucopy_ATHLON, qtrmm_oltncopy_ATHLON,
#endif
  qtrmm_ounucopy_ATHLON, qtrmm_ounncopy_ATHLON, qtrmm_outucopy_ATHLON, qtrmm_outncopy_ATHLON,
  qtrmm_olnucopy_ATHLON, qtrmm_olnncopy_ATHLON, qtrmm_oltucopy_ATHLON, qtrmm_oltncopy_ATHLON,
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
  qsymm_iutcopy_ATHLON, qsymm_iltcopy_ATHLON,
#else
  qsymm_outcopy_ATHLON, qsymm_oltcopy_ATHLON,
#endif
  qsymm_outcopy_ATHLON, qsymm_oltcopy_ATHLON,

#ifndef NO_LAPACK
  qneg_tcopy_ATHLON, qlaswp_ncopy_ATHLON,
#else
  NULL, NULL,
#endif

#endif

#if (BUILD_COMPLEX || BUILD_COMPLEX16)
  0, 0, 0,
  CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N,
#ifdef CGEMM_DEFAULT_UNROLL_MN
 CGEMM_DEFAULT_UNROLL_MN,
#else
 MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N),
#endif
  camax_k_ATHLON, camin_k_ATHLON, icamax_k_ATHLON, icamin_k_ATHLON,
#endif
#if (BUILD_COMPLEX)
  cnrm2_k_ATHLON, casum_k_ATHLON, csum_k_ATHLON,
#endif
#if (BUILD_COMPLEX || BUILD_COMPLEX16)
  ccopy_k_ATHLON,  cdotu_k_ATHLON, cdotc_k_ATHLON,
#endif
#if (BUILD_COMPLEX)
 csrot_k_ATHLON,
#endif
#if (BUILD_COMPLEX || BUILD_COMPLEX16)
  caxpy_k_ATHLON,
  caxpyc_k_ATHLON, 
  cscal_k_ATHLON, 
  cswap_k_ATHLON,

  cgemv_n_ATHLON, cgemv_t_ATHLON, cgemv_r_ATHLON, cgemv_c_ATHLON,
  cgemv_o_ATHLON, cgemv_u_ATHLON, cgemv_s_ATHLON, cgemv_d_ATHLON,
#endif
#if (BUILD_COMPLEX)
  cgeru_k_ATHLON, cgerc_k_ATHLON, cgerv_k_ATHLON, cgerd_k_ATHLON,
  csymv_L_ATHLON, csymv_U_ATHLON,
  chemv_L_ATHLON, chemv_U_ATHLON, chemv_M_ATHLON, chemv_V_ATHLON,
#endif
#if (BUILD_COMPLEX || BUILD_COMPLEX16)
  cgemm_kernel_n_ATHLON, cgemm_kernel_l_ATHLON, cgemm_kernel_r_ATHLON, cgemm_kernel_b_ATHLON,
  cgemm_beta_ATHLON,
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
  cgemm_incopy_ATHLON, cgemm_itcopy_ATHLON,
#else
  cgemm_oncopy_ATHLON, cgemm_otcopy_ATHLON,
#endif
  cgemm_oncopy_ATHLON, cgemm_otcopy_ATHLON,

#ifdef SMALL_MATRIX_OPT
  cgemm_small_matrix_permit_ATHLON,
  cgemm_small_kernel_nn_ATHLON, cgemm_small_kernel_nt_ATHLON, cgemm_small_kernel_nr_ATHLON, cgemm_small_kernel_nc_ATHLON,
  cgemm_small_kernel_tn_ATHLON, cgemm_small_kernel_tt_ATHLON, cgemm_small_kernel_tr_ATHLON, cgemm_small_kernel_tc_ATHLON,
  cgemm_small_kernel_rn_ATHLON, cgemm_small_kernel_rt_ATHLON, cgemm_small_kernel_rr_ATHLON, cgemm_small_kernel_rc_ATHLON,
  cgemm_small_kernel_cn_ATHLON, cgemm_small_kernel_ct_ATHLON, cgemm_small_kernel_cr_ATHLON, cgemm_small_kernel_cc_ATHLON,
  cgemm_small_kernel_b0_nn_ATHLON, cgemm_small_kernel_b0_nt_ATHLON, cgemm_small_kernel_b0_nr_ATHLON, cgemm_small_kernel_b0_nc_ATHLON,
  cgemm_small_kernel_b0_tn_ATHLON, cgemm_small_kernel_b0_tt_ATHLON, cgemm_small_kernel_b0_tr_ATHLON, cgemm_small_kernel_b0_tc_ATHLON,
  cgemm_small_kernel_b0_rn_ATHLON, cgemm_small_kernel_b0_rt_ATHLON, cgemm_small_kernel_b0_rr_ATHLON, cgemm_small_kernel_b0_rc_ATHLON,
  cgemm_small_kernel_b0_cn_ATHLON, cgemm_small_kernel_b0_ct_ATHLON, cgemm_small_kernel_b0_cr_ATHLON, cgemm_small_kernel_b0_cc_ATHLON,
#endif

  ctrsm_kernel_LN_ATHLON, ctrsm_kernel_LT_ATHLON, ctrsm_kernel_LR_ATHLON, ctrsm_kernel_LC_ATHLON,
  ctrsm_kernel_RN_ATHLON, ctrsm_kernel_RT_ATHLON, ctrsm_kernel_RR_ATHLON, ctrsm_kernel_RC_ATHLON,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
  ctrsm_iunucopy_ATHLON,  ctrsm_iunncopy_ATHLON,  ctrsm_iutucopy_ATHLON,  ctrsm_iutncopy_ATHLON,
  ctrsm_ilnucopy_ATHLON,  ctrsm_ilnncopy_ATHLON,  ctrsm_iltucopy_ATHLON,  ctrsm_iltncopy_ATHLON,
#else
  ctrsm_ounucopy_ATHLON,  ctrsm_ounncopy_ATHLON,  ctrsm_outucopy_ATHLON,  ctrsm_outncopy_ATHLON,
  ctrsm_olnucopy_ATHLON,  ctrsm_olnncopy_ATHLON,  ctrsm_oltucopy_ATHLON,  ctrsm_oltncopy_ATHLON,
#endif
  ctrsm_ounucopy_ATHLON,  ctrsm_ounncopy_ATHLON,  ctrsm_outucopy_ATHLON,  ctrsm_outncopy_ATHLON,
  ctrsm_olnucopy_ATHLON,  ctrsm_olnncopy_ATHLON,  ctrsm_oltucopy_ATHLON,  ctrsm_oltncopy_ATHLON,
#endif
#if (BUILD_COMPLEX)

  ctrmm_kernel_RN_ATHLON,  ctrmm_kernel_RT_ATHLON,  ctrmm_kernel_RR_ATHLON,  ctrmm_kernel_RC_ATHLON,
  ctrmm_kernel_LN_ATHLON,  ctrmm_kernel_LT_ATHLON,  ctrmm_kernel_LR_ATHLON,  ctrmm_kernel_LC_ATHLON,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
  ctrmm_iunucopy_ATHLON,  ctrmm_iunncopy_ATHLON,  ctrmm_iutucopy_ATHLON,  ctrmm_iutncopy_ATHLON,
  ctrmm_ilnucopy_ATHLON,  ctrmm_ilnncopy_ATHLON,  ctrmm_iltucopy_ATHLON,  ctrmm_iltncopy_ATHLON,
#else
  ctrmm_ounucopy_ATHLON,  ctrmm_ounncopy_ATHLON,  ctrmm_outucopy_ATHLON,  ctrmm_outncopy_ATHLON,
  ctrmm_olnucopy_ATHLON,  ctrmm_olnncopy_ATHLON,  ctrmm_oltucopy_ATHLON,  ctrmm_oltncopy_ATHLON,
#endif
  ctrmm_ounucopy_ATHLON,  ctrmm_ounncopy_ATHLON,  ctrmm_outucopy_ATHLON,  ctrmm_outncopy_ATHLON,
  ctrmm_olnucopy_ATHLON,  ctrmm_olnncopy_ATHLON,  ctrmm_oltucopy_ATHLON,  ctrmm_oltncopy_ATHLON,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
  csymm_iutcopy_ATHLON,  csymm_iltcopy_ATHLON,
#else
  csymm_outcopy_ATHLON,  csymm_oltcopy_ATHLON,
#endif
  csymm_outcopy_ATHLON,  csymm_oltcopy_ATHLON,
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
  chemm_iutcopy_ATHLON,  chemm_iltcopy_ATHLON,
#else
  chemm_outcopy_ATHLON,  chemm_oltcopy_ATHLON,
#endif
  chemm_outcopy_ATHLON,  chemm_oltcopy_ATHLON,

  0, 0, 0,

#if (USE_GEMM3M)
#ifdef CGEMM3M_DEFAULT_UNROLL_M
  CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N, MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N),
#else
  SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),
#endif


  cgemm3m_kernel_ATHLON,

  cgemm3m_incopyb_ATHLON,  cgemm3m_incopyr_ATHLON,
  cgemm3m_incopyi_ATHLON,  cgemm3m_itcopyb_ATHLON,
  cgemm3m_itcopyr_ATHLON,  cgemm3m_itcopyi_ATHLON,
  cgemm3m_oncopyb_ATHLON,  cgemm3m_oncopyr_ATHLON,
  cgemm3m_oncopyi_ATHLON,  cgemm3m_otcopyb_ATHLON,
  cgemm3m_otcopyr_ATHLON,  cgemm3m_otcopyi_ATHLON,

  csymm3m_iucopyb_ATHLON,  csymm3m_ilcopyb_ATHLON,
  csymm3m_iucopyr_ATHLON,  csymm3m_ilcopyr_ATHLON,
  csymm3m_iucopyi_ATHLON,  csymm3m_ilcopyi_ATHLON,
  csymm3m_oucopyb_ATHLON,  csymm3m_olcopyb_ATHLON,
  csymm3m_oucopyr_ATHLON,  csymm3m_olcopyr_ATHLON,
  csymm3m_oucopyi_ATHLON,  csymm3m_olcopyi_ATHLON,

  chemm3m_iucopyb_ATHLON,  chemm3m_ilcopyb_ATHLON,
  chemm3m_iucopyr_ATHLON,  chemm3m_ilcopyr_ATHLON,
  chemm3m_iucopyi_ATHLON,  chemm3m_ilcopyi_ATHLON,

  chemm3m_oucopyb_ATHLON,  chemm3m_olcopyb_ATHLON,
  chemm3m_oucopyr_ATHLON,  chemm3m_olcopyr_ATHLON,
  chemm3m_oucopyi_ATHLON,  chemm3m_olcopyi_ATHLON,
#else
  0, 0, 0,

  NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
#endif
#endif

#if (BUILD_COMPLEX || BUILD_COMPLEX16)
#ifndef NO_LAPACK
  cneg_tcopy_ATHLON,
  
   claswp_ncopy_ATHLON,
#else
  NULL, NULL,
#endif

#endif

#if BUILD_COMPLEX16 == 1
  0, 0, 0,
  ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N,
#ifdef ZGEMM_DEFAULT_UNROLL_MN
 ZGEMM_DEFAULT_UNROLL_MN,
#else
 MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N),
#endif

  zamax_k_ATHLON, zamin_k_ATHLON, izamax_k_ATHLON, izamin_k_ATHLON,
  znrm2_k_ATHLON, zasum_k_ATHLON, zsum_k_ATHLON, zcopy_k_ATHLON,
  zdotu_k_ATHLON, zdotc_k_ATHLON, zdrot_k_ATHLON,
  zaxpy_k_ATHLON, zaxpyc_k_ATHLON, zscal_k_ATHLON, zswap_k_ATHLON,

  zgemv_n_ATHLON, zgemv_t_ATHLON, zgemv_r_ATHLON, zgemv_c_ATHLON,
  zgemv_o_ATHLON, zgemv_u_ATHLON, zgemv_s_ATHLON, zgemv_d_ATHLON,
  zgeru_k_ATHLON, zgerc_k_ATHLON, zgerv_k_ATHLON, zgerd_k_ATHLON,
  zsymv_L_ATHLON, zsymv_U_ATHLON,
  zhemv_L_ATHLON, zhemv_U_ATHLON, zhemv_M_ATHLON, zhemv_V_ATHLON,

  zgemm_kernel_n_ATHLON, zgemm_kernel_l_ATHLON, zgemm_kernel_r_ATHLON, zgemm_kernel_b_ATHLON,
  zgemm_beta_ATHLON,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
  zgemm_incopy_ATHLON, zgemm_itcopy_ATHLON,
#else
  zgemm_oncopy_ATHLON, zgemm_otcopy_ATHLON,
#endif
  zgemm_oncopy_ATHLON, zgemm_otcopy_ATHLON,

#ifdef SMALL_MATRIX_OPT
  zgemm_small_matrix_permit_ATHLON,
  zgemm_small_kernel_nn_ATHLON, zgemm_small_kernel_nt_ATHLON, zgemm_small_kernel_nr_ATHLON, zgemm_small_kernel_nc_ATHLON,
  zgemm_small_kernel_tn_ATHLON, zgemm_small_kernel_tt_ATHLON, zgemm_small_kernel_tr_ATHLON, zgemm_small_kernel_tc_ATHLON,
  zgemm_small_kernel_rn_ATHLON, zgemm_small_kernel_rt_ATHLON, zgemm_small_kernel_rr_ATHLON, zgemm_small_kernel_rc_ATHLON,
  zgemm_small_kernel_cn_ATHLON, zgemm_small_kernel_ct_ATHLON, zgemm_small_kernel_cr_ATHLON, zgemm_small_kernel_cc_ATHLON,
  zgemm_small_kernel_b0_nn_ATHLON, zgemm_small_kernel_b0_nt_ATHLON, zgemm_small_kernel_b0_nr_ATHLON, zgemm_small_kernel_b0_nc_ATHLON,
  zgemm_small_kernel_b0_tn_ATHLON, zgemm_small_kernel_b0_tt_ATHLON, zgemm_small_kernel_b0_tr_ATHLON, zgemm_small_kernel_b0_tc_ATHLON,
  zgemm_small_kernel_b0_rn_ATHLON, zgemm_small_kernel_b0_rt_ATHLON, zgemm_small_kernel_b0_rr_ATHLON, zgemm_small_kernel_b0_rc_ATHLON,
  zgemm_small_kernel_b0_cn_ATHLON, zgemm_small_kernel_b0_ct_ATHLON, zgemm_small_kernel_b0_cr_ATHLON, zgemm_small_kernel_b0_cc_ATHLON,
#endif

  ztrsm_kernel_LN_ATHLON, ztrsm_kernel_LT_ATHLON, ztrsm_kernel_LR_ATHLON, ztrsm_kernel_LC_ATHLON,
  ztrsm_kernel_RN_ATHLON, ztrsm_kernel_RT_ATHLON, ztrsm_kernel_RR_ATHLON, ztrsm_kernel_RC_ATHLON,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
  ztrsm_iunucopy_ATHLON,  ztrsm_iunncopy_ATHLON,  ztrsm_iutucopy_ATHLON,  ztrsm_iutncopy_ATHLON,
  ztrsm_ilnucopy_ATHLON,  ztrsm_ilnncopy_ATHLON,  ztrsm_iltucopy_ATHLON,  ztrsm_iltncopy_ATHLON,
#else
  ztrsm_ounucopy_ATHLON,  ztrsm_ounncopy_ATHLON,  ztrsm_outucopy_ATHLON,  ztrsm_outncopy_ATHLON,
  ztrsm_olnucopy_ATHLON,  ztrsm_olnncopy_ATHLON,  ztrsm_oltucopy_ATHLON,  ztrsm_oltncopy_ATHLON,
#endif
  ztrsm_ounucopy_ATHLON,  ztrsm_ounncopy_ATHLON,  ztrsm_outucopy_ATHLON,  ztrsm_outncopy_ATHLON,
  ztrsm_olnucopy_ATHLON,  ztrsm_olnncopy_ATHLON,  ztrsm_oltucopy_ATHLON,  ztrsm_oltncopy_ATHLON,

  ztrmm_kernel_RN_ATHLON,  ztrmm_kernel_RT_ATHLON,  ztrmm_kernel_RR_ATHLON,  ztrmm_kernel_RC_ATHLON,
  ztrmm_kernel_LN_ATHLON,  ztrmm_kernel_LT_ATHLON,  ztrmm_kernel_LR_ATHLON,  ztrmm_kernel_LC_ATHLON,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
  ztrmm_iunucopy_ATHLON,  ztrmm_iunncopy_ATHLON,  ztrmm_iutucopy_ATHLON,  ztrmm_iutncopy_ATHLON,
  ztrmm_ilnucopy_ATHLON,  ztrmm_ilnncopy_ATHLON,  ztrmm_iltucopy_ATHLON,  ztrmm_iltncopy_ATHLON,
#else
  ztrmm_ounucopy_ATHLON,  ztrmm_ounncopy_ATHLON,  ztrmm_outucopy_ATHLON,  ztrmm_outncopy_ATHLON,
  ztrmm_olnucopy_ATHLON,  ztrmm_olnncopy_ATHLON,  ztrmm_oltucopy_ATHLON,  ztrmm_oltncopy_ATHLON,
#endif
  ztrmm_ounucopy_ATHLON,  ztrmm_ounncopy_ATHLON,  ztrmm_outucopy_ATHLON,  ztrmm_outncopy_ATHLON,
  ztrmm_olnucopy_ATHLON,  ztrmm_olnncopy_ATHLON,  ztrmm_oltucopy_ATHLON,  ztrmm_oltncopy_ATHLON,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
  zsymm_iutcopy_ATHLON,  zsymm_iltcopy_ATHLON,
#else
  zsymm_outcopy_ATHLON,  zsymm_oltcopy_ATHLON,
#endif
  zsymm_outcopy_ATHLON,  zsymm_oltcopy_ATHLON,
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
  zhemm_iutcopy_ATHLON,  zhemm_iltcopy_ATHLON,
#else
  zhemm_outcopy_ATHLON,  zhemm_oltcopy_ATHLON,
#endif
  zhemm_outcopy_ATHLON,  zhemm_oltcopy_ATHLON,

  0, 0, 0,
#if (USE_GEMM3M)
#ifdef ZGEMM3M_DEFAULT_UNROLL_M
  ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N, MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N),
#else
  DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),
#endif


  zgemm3m_kernel_ATHLON,

  zgemm3m_incopyb_ATHLON,  zgemm3m_incopyr_ATHLON,
  zgemm3m_incopyi_ATHLON,  zgemm3m_itcopyb_ATHLON,
  zgemm3m_itcopyr_ATHLON,  zgemm3m_itcopyi_ATHLON,
  zgemm3m_oncopyb_ATHLON,  zgemm3m_oncopyr_ATHLON,
  zgemm3m_oncopyi_ATHLON,  zgemm3m_otcopyb_ATHLON,
  zgemm3m_otcopyr_ATHLON,  zgemm3m_otcopyi_ATHLON,

  zsymm3m_iucopyb_ATHLON,  zsymm3m_ilcopyb_ATHLON,
  zsymm3m_iucopyr_ATHLON,  zsymm3m_ilcopyr_ATHLON,
  zsymm3m_iucopyi_ATHLON,  zsymm3m_ilcopyi_ATHLON,
  zsymm3m_oucopyb_ATHLON,  zsymm3m_olcopyb_ATHLON,
  zsymm3m_oucopyr_ATHLON,  zsymm3m_olcopyr_ATHLON,
  zsymm3m_oucopyi_ATHLON,  zsymm3m_olcopyi_ATHLON,

  zhemm3m_iucopyb_ATHLON,  zhemm3m_ilcopyb_ATHLON,
  zhemm3m_iucopyr_ATHLON,  zhemm3m_ilcopyr_ATHLON,
  zhemm3m_iucopyi_ATHLON,  zhemm3m_ilcopyi_ATHLON,

  zhemm3m_oucopyb_ATHLON,  zhemm3m_olcopyb_ATHLON,
  zhemm3m_oucopyr_ATHLON,  zhemm3m_olcopyr_ATHLON,
  zhemm3m_oucopyi_ATHLON,  zhemm3m_olcopyi_ATHLON,
#else
  0, 0, 0,

  NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
#endif

#ifndef NO_LAPACK
  zneg_tcopy_ATHLON, zlaswp_ncopy_ATHLON,
#else
  NULL, NULL,
#endif

#endif

#ifdef EXPRECISION

  0, 0, 0,
  XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N, MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N),

  xamax_k_ATHLON, xamin_k_ATHLON, ixamax_k_ATHLON, ixamin_k_ATHLON,
  xnrm2_k_ATHLON, xasum_k_ATHLON, xsum_k_ATHLON, xcopy_k_ATHLON,
  xdotu_k_ATHLON, xdotc_k_ATHLON, xqrot_k_ATHLON,
  xaxpy_k_ATHLON, xaxpyc_k_ATHLON, xscal_k_ATHLON, xswap_k_ATHLON,

  xgemv_n_ATHLON, xgemv_t_ATHLON, xgemv_r_ATHLON, xgemv_c_ATHLON,
  xgemv_o_ATHLON, xgemv_u_ATHLON, xgemv_s_ATHLON, xgemv_d_ATHLON,
  xgeru_k_ATHLON, xgerc_k_ATHLON, xgerv_k_ATHLON, xgerd_k_ATHLON,
  xsymv_L_ATHLON, xsymv_U_ATHLON,
  xhemv_L_ATHLON, xhemv_U_ATHLON, xhemv_M_ATHLON, xhemv_V_ATHLON,

  xgemm_kernel_n_ATHLON, xgemm_kernel_l_ATHLON, xgemm_kernel_r_ATHLON, xgemm_kernel_b_ATHLON,
  xgemm_beta_ATHLON,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
  xgemm_incopy_ATHLON, xgemm_itcopy_ATHLON,
#else
  xgemm_oncopy_ATHLON, xgemm_otcopy_ATHLON,
#endif
  xgemm_oncopy_ATHLON, xgemm_otcopy_ATHLON,

  xtrsm_kernel_LN_ATHLON, xtrsm_kernel_LT_ATHLON, xtrsm_kernel_LR_ATHLON, xtrsm_kernel_LC_ATHLON,
  xtrsm_kernel_RN_ATHLON, xtrsm_kernel_RT_ATHLON, xtrsm_kernel_RR_ATHLON, xtrsm_kernel_RC_ATHLON,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
  xtrsm_iunucopy_ATHLON,  xtrsm_iunncopy_ATHLON,  xtrsm_iutucopy_ATHLON,  xtrsm_iutncopy_ATHLON,
  xtrsm_ilnucopy_ATHLON,  xtrsm_ilnncopy_ATHLON,  xtrsm_iltucopy_ATHLON,  xtrsm_iltncopy_ATHLON,
#else
  xtrsm_ounucopy_ATHLON,  xtrsm_ounncopy_ATHLON,  xtrsm_outucopy_ATHLON,  xtrsm_outncopy_ATHLON,
  xtrsm_olnucopy_ATHLON,  xtrsm_olnncopy_ATHLON,  xtrsm_oltucopy_ATHLON,  xtrsm_oltncopy_ATHLON,
#endif
  xtrsm_ounucopy_ATHLON,  xtrsm_ounncopy_ATHLON,  xtrsm_outucopy_ATHLON,  xtrsm_outncopy_ATHLON,
  xtrsm_olnucopy_ATHLON,  xtrsm_olnncopy_ATHLON,  xtrsm_oltucopy_ATHLON,  xtrsm_oltncopy_ATHLON,

  xtrmm_kernel_RN_ATHLON,  xtrmm_kernel_RT_ATHLON,  xtrmm_kernel_RR_ATHLON,  xtrmm_kernel_RC_ATHLON,
  xtrmm_kernel_LN_ATHLON,  xtrmm_kernel_LT_ATHLON,  xtrmm_kernel_LR_ATHLON,  xtrmm_kernel_LC_ATHLON,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
  xtrmm_iunucopy_ATHLON,  xtrmm_iunncopy_ATHLON,  xtrmm_iutucopy_ATHLON,  xtrmm_iutncopy_ATHLON,
  xtrmm_ilnucopy_ATHLON,  xtrmm_ilnncopy_ATHLON,  xtrmm_iltucopy_ATHLON,  xtrmm_iltncopy_ATHLON,
#else
  xtrmm_ounucopy_ATHLON,  xtrmm_ounncopy_ATHLON,  xtrmm_outucopy_ATHLON,  xtrmm_outncopy_ATHLON,
  xtrmm_olnucopy_ATHLON,  xtrmm_olnncopy_ATHLON,  xtrmm_oltucopy_ATHLON,  xtrmm_oltncopy_ATHLON,
#endif
  xtrmm_ounucopy_ATHLON,  xtrmm_ounncopy_ATHLON,  xtrmm_outucopy_ATHLON,  xtrmm_outncopy_ATHLON,
  xtrmm_olnucopy_ATHLON,  xtrmm_olnncopy_ATHLON,  xtrmm_oltucopy_ATHLON,  xtrmm_oltncopy_ATHLON,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
  xsymm_iutcopy_ATHLON,  xsymm_iltcopy_ATHLON,
#else
  xsymm_outcopy_ATHLON,  xsymm_oltcopy_ATHLON,
#endif
  xsymm_outcopy_ATHLON,  xsymm_oltcopy_ATHLON,
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
  xhemm_iutcopy_ATHLON,  xhemm_iltcopy_ATHLON,
#else
  xhemm_outcopy_ATHLON,  xhemm_oltcopy_ATHLON,
#endif
  xhemm_outcopy_ATHLON,  xhemm_oltcopy_ATHLON,

  0, 0, 0,
#if (USE_GEMM3M)
  QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N),

  xgemm3m_kernel_ATHLON,

  xgemm3m_incopyb_ATHLON,  xgemm3m_incopyr_ATHLON,
  xgemm3m_incopyi_ATHLON,  xgemm3m_itcopyb_ATHLON,
  xgemm3m_itcopyr_ATHLON,  xgemm3m_itcopyi_ATHLON,
  xgemm3m_oncopyb_ATHLON,  xgemm3m_oncopyr_ATHLON,
  xgemm3m_oncopyi_ATHLON,  xgemm3m_otcopyb_ATHLON,
  xgemm3m_otcopyr_ATHLON,  xgemm3m_otcopyi_ATHLON,

  xsymm3m_iucopyb_ATHLON,  xsymm3m_ilcopyb_ATHLON,
  xsymm3m_iucopyr_ATHLON,  xsymm3m_ilcopyr_ATHLON,
  xsymm3m_iucopyi_ATHLON,  xsymm3m_ilcopyi_ATHLON,
  xsymm3m_oucopyb_ATHLON,  xsymm3m_olcopyb_ATHLON,
  xsymm3m_oucopyr_ATHLON,  xsymm3m_olcopyr_ATHLON,
  xsymm3m_oucopyi_ATHLON,  xsymm3m_olcopyi_ATHLON,

  xhemm3m_iucopyb_ATHLON,  xhemm3m_ilcopyb_ATHLON,
  xhemm3m_iucopyr_ATHLON,  xhemm3m_ilcopyr_ATHLON,
  xhemm3m_iucopyi_ATHLON,  xhemm3m_ilcopyi_ATHLON,

  xhemm3m_oucopyb_ATHLON,  xhemm3m_olcopyb_ATHLON,
  xhemm3m_oucopyr_ATHLON,  xhemm3m_olcopyr_ATHLON,
  xhemm3m_oucopyi_ATHLON,  xhemm3m_olcopyi_ATHLON,
#else
  0, 0, 0,

  NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,

  NULL, NULL,
  NULL, NULL,
  NULL, NULL,
#endif

#ifndef NO_LAPACK
  xneg_tcopy_ATHLON, xlaswp_ncopy_ATHLON,
#else
  NULL, NULL,
#endif

#endif

  init_parameter,

  SNUMOPT, DNUMOPT, QNUMOPT,
#if BUILD_SINGLE == 1
  saxpby_k_ATHLON,
#endif
#if BUILD_DOUBLE  == 1
  daxpby_k_ATHLON,
#endif
#if BUILD_COMPLEX == 1
  caxpby_k_ATHLON,
#endif
#if BUILD_COMPLEX16== 1
  zaxpby_k_ATHLON,
#endif

#if BUILD_SINGLE == 1
  somatcopy_k_cn_ATHLON, somatcopy_k_ct_ATHLON, somatcopy_k_rn_ATHLON, somatcopy_k_rt_ATHLON,
#endif
#if BUILD_DOUBLE== 1
  domatcopy_k_cn_ATHLON, domatcopy_k_ct_ATHLON, domatcopy_k_rn_ATHLON, domatcopy_k_rt_ATHLON,
#endif
#if BUILD_COMPLEX == 1
  comatcopy_k_cn_ATHLON, comatcopy_k_ct_ATHLON, comatcopy_k_rn_ATHLON, comatcopy_k_rt_ATHLON,
  comatcopy_k_cnc_ATHLON, comatcopy_k_ctc_ATHLON, comatcopy_k_rnc_ATHLON, comatcopy_k_rtc_ATHLON,
#endif
#if BUILD_COMPLEX16 == 1
  zomatcopy_k_cn_ATHLON, zomatcopy_k_ct_ATHLON, zomatcopy_k_rn_ATHLON, zomatcopy_k_rt_ATHLON,
  zomatcopy_k_cnc_ATHLON, zomatcopy_k_ctc_ATHLON, zomatcopy_k_rnc_ATHLON, zomatcopy_k_rtc_ATHLON,
#endif

#if BUILD_SINGLE == 1
  simatcopy_k_cn_ATHLON, simatcopy_k_ct_ATHLON, simatcopy_k_rn_ATHLON, simatcopy_k_rt_ATHLON,
#endif
#if BUILD_DOUBLE== 1
  dimatcopy_k_cn_ATHLON, dimatcopy_k_ct_ATHLON, dimatcopy_k_rn_ATHLON, dimatcopy_k_rt_ATHLON,
#endif
#if BUILD_COMPLEX== 1
  cimatcopy_k_cn_ATHLON, cimatcopy_k_ct_ATHLON, cimatcopy_k_rn_ATHLON, cimatcopy_k_rt_ATHLON,
  cimatcopy_k_cnc_ATHLON, cimatcopy_k_ctc_ATHLON, cimatcopy_k_rnc_ATHLON, cimatcopy_k_rtc_ATHLON,
#endif
#if BUILD_COMPLEX16==1
  zimatcopy_k_cn_ATHLON, zimatcopy_k_ct_ATHLON, zimatcopy_k_rn_ATHLON, zimatcopy_k_rt_ATHLON,
  zimatcopy_k_cnc_ATHLON, zimatcopy_k_ctc_ATHLON, zimatcopy_k_rnc_ATHLON, zimatcopy_k_rtc_ATHLON,
#endif

#if BUILD_SINGLE == 1
  sgeadd_k_ATHLON,
#endif
#if BUILD_DOUBLE==1
  dgeadd_k_ATHLON,
#endif
#if BUILD_COMPLEX==1
  cgeadd_k_ATHLON,
#endif
#if BUILD_COMPLEX16==1
  zgeadd_k_ATHLON
#endif
};

#if (ARCH_ARM64)
static void init_parameter(void) {
#if (BUILD_BFLOAT16)
  TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
#endif
#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE == 1
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif

#if (BUILD_BFLOAT16)
  TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
#endif
#if BUILD_SINGLE == 1
  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
#endif
#if BUILD_DOUBLE== 1
  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
#endif
#if BUILD_COMPLEX== 1
  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
#endif

#if (BUILD_BFLOAT16)
  TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
#endif
#if BUILD_SINGLE == 1
  TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
#endif
#if BUILD_DOUBLE==1 
  TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;
#endif

#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
  TABLE_NAME.qgemm_q = QGEMM_DEFAULT_Q;
  TABLE_NAME.xgemm_q = XGEMM_DEFAULT_Q;
  TABLE_NAME.qgemm_r = QGEMM_DEFAULT_R;
  TABLE_NAME.xgemm_r = XGEMM_DEFAULT_R;
#endif

#if (USE_GEMM3M)
#ifdef CGEMM3M_DEFAULT_P
  TABLE_NAME.cgemm3m_p = CGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.cgemm3m_p = TABLE_NAME.sgemm_p;
#endif

#ifdef ZGEMM3M_DEFAULT_P
  TABLE_NAME.zgemm3m_p = ZGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.zgemm3m_p = TABLE_NAME.dgemm_p;
#endif

#ifdef CGEMM3M_DEFAULT_Q
  TABLE_NAME.cgemm3m_q = CGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.cgemm3m_q = TABLE_NAME.sgemm_q;
#endif

#ifdef ZGEMM3M_DEFAULT_Q
  TABLE_NAME.zgemm3m_q = ZGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.zgemm3m_q = TABLE_NAME.dgemm_q;
#endif

#ifdef CGEMM3M_DEFAULT_R
  TABLE_NAME.cgemm3m_r = CGEMM3M_DEFAULT_R;
#else
  TABLE_NAME.cgemm3m_r = TABLE_NAME.sgemm_r;
#endif

#ifdef ZGEMM3M_DEFAULT_R
  TABLE_NAME.zgemm3m_r = ZGEMM3M_DEFAULT_R;
#else
  TABLE_NAME.zgemm3m_r = TABLE_NAME.dgemm_r;
#endif

#ifdef EXPRECISION
  TABLE_NAME.xgemm3m_p = TABLE_NAME.qgemm_p;
  TABLE_NAME.xgemm3m_q = TABLE_NAME.qgemm_q;
  TABLE_NAME.xgemm3m_r = TABLE_NAME.qgemm_r;
#endif
#endif

}
#else // (ARCH_ARM64)
#if defined(ARCH_MIPS64)
static void init_parameter(void) {
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;

  TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
  TABLE_NAME.dgemm_r = 640;
  TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
  TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;

#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
  TABLE_NAME.qgemm_q = QGEMM_DEFAULT_Q;
  TABLE_NAME.xgemm_q = XGEMM_DEFAULT_Q;
  TABLE_NAME.qgemm_r = QGEMM_DEFAULT_R;
  TABLE_NAME.xgemm_r = XGEMM_DEFAULT_R;
#endif

#if defined(USE_GEMM3M)
#ifdef CGEMM3M_DEFAULT_P
  TABLE_NAME.cgemm3m_p = CGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.cgemm3m_p = TABLE_NAME.sgemm_p;
#endif

#ifdef ZGEMM3M_DEFAULT_P
  TABLE_NAME.zgemm3m_p = ZGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.zgemm3m_p = TABLE_NAME.dgemm_p;
#endif

#ifdef CGEMM3M_DEFAULT_Q
  TABLE_NAME.cgemm3m_q = CGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.cgemm3m_q = TABLE_NAME.sgemm_q;
#endif

#ifdef ZGEMM3M_DEFAULT_Q
  TABLE_NAME.zgemm3m_q = ZGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.zgemm3m_q = TABLE_NAME.dgemm_q;
#endif

#ifdef CGEMM3M_DEFAULT_R
  TABLE_NAME.cgemm3m_r = CGEMM3M_DEFAULT_R;
#else
  TABLE_NAME.cgemm3m_r = TABLE_NAME.sgemm_r;
#endif

#ifdef ZGEMM3M_DEFAULT_R
  TABLE_NAME.zgemm3m_r = ZGEMM3M_DEFAULT_R;
#else
  TABLE_NAME.zgemm3m_r = TABLE_NAME.dgemm_r;
#endif

#ifdef EXPRECISION
  TABLE_NAME.xgemm3m_p = TABLE_NAME.qgemm_p;
  TABLE_NAME.xgemm3m_q = TABLE_NAME.qgemm_q;
  TABLE_NAME.xgemm3m_r = TABLE_NAME.qgemm_r;
#endif
#endif
}
#else // (ARCH_MIPS64)
#if (ARCH_POWER)
static void init_parameter(void) {

#ifdef BUILD_BFLOAT16
  TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
#endif
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

#ifdef BUILD_BFLOAT16
  TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
#endif
  TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
  TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
  TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
  TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;


#ifdef BUILD_BFLOAT16
  TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
#endif
  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
}
#else //POWER

#if (ARCH_ZARCH)
static void init_parameter(void) {
#ifdef BUILD_BFLOAT16
	TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
#endif
	TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
	TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
	TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
	TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

#ifdef BUILD_BFLOAT16
	TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
#endif
	TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
	TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
	TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
	TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;


#ifdef BUILD_BFLOAT16
	TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
#endif
	TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
	TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
	TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
	TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
}
#else //ZARCH

#ifdef ARCH_X86
static int get_l2_size_old(void){
  int i, eax, ebx, ecx, edx, cpuid_level;
  int info[15];

  cpuid(2, &eax, &ebx, &ecx, &edx);

  info[ 0] = BITMASK(eax,  8, 0xff);
  info[ 1] = BITMASK(eax, 16, 0xff);
  info[ 2] = BITMASK(eax, 24, 0xff);

  info[ 3] = BITMASK(ebx,  0, 0xff);
  info[ 4] = BITMASK(ebx,  8, 0xff);
  info[ 5] = BITMASK(ebx, 16, 0xff);
  info[ 6] = BITMASK(ebx, 24, 0xff);

  info[ 7] = BITMASK(ecx,  0, 0xff);
  info[ 8] = BITMASK(ecx,  8, 0xff);
  info[ 9] = BITMASK(ecx, 16, 0xff);
  info[10] = BITMASK(ecx, 24, 0xff);

  info[11] = BITMASK(edx,  0, 0xff);
  info[12] = BITMASK(edx,  8, 0xff);
  info[13] = BITMASK(edx, 16, 0xff);
  info[14] = BITMASK(edx, 24, 0xff);

  for (i = 0; i < 15; i++){

    switch (info[i]){

      /* This table is from http://www.sandpile.org/ia32/cpuid.htm */

    case 0x1a :
      return 96;

    case 0x39 :
    case 0x3b :
    case 0x41 :
    case 0x79 :
    case 0x81 :
      return 128;

    case 0x3a :
      return 192;

    case 0x21 :
    case 0x3c :
    case 0x42 :
    case 0x7a :
    case 0x7e :
    case 0x82 :
      return 256;

    case 0x3d :
      return 384;

    case 0x3e :
    case 0x43 :
    case 0x7b :
    case 0x7f :
    case 0x83 :
    case 0x86 :
      return 512;

    case 0x44 :
    case 0x78 :
    case 0x7c :
    case 0x84 :
    case 0x87 :
      return 1024;

    case 0x45 :
    case 0x7d :
    case 0x85 :
      return 2048;

    case 0x48 :
      return 3184;

    case 0x49 :
      return 4096;

    case 0x4e :
      return 6144;
    }
  }
//  return 0;
fprintf (stderr,"OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k\n");
return 256;
}
#endif

static __inline__ int get_l2_size(void){

  int eax, ebx, ecx, edx, l2;

  cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

  l2 = BITMASK(ecx, 16, 0xffff);

#ifndef ARCH_X86
  if (l2 <= 0) {
     fprintf (stderr,"OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k\n");
     return 256;
  }
  return l2;

#else

  if (l2 > 0) return l2;

  return get_l2_size_old();
#endif
}

static __inline__ int get_l3_size(void){

  int eax, ebx, ecx, edx;

  cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

  return BITMASK(edx, 18, 0x3fff) * 512;
}


static void init_parameter(void) {

  int l2 = get_l2_size();

  (void) l2; /* dirty trick to suppress unused variable warning for targets */
             /* where the GEMM unrolling parameters do not depend on l2 */
  
#ifdef BUILD_BFLOAT16
  TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
  TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
  TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
#endif
#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
#endif
#if  (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)
  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
#endif
#if BUILD_COMPLEX == 1
  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
#endif

#if BUILD_COMPLEX == 1
#ifdef CGEMM3M_DEFAULT_Q
  TABLE_NAME.cgemm3m_q = CGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.cgemm3m_q = SGEMM_DEFAULT_Q;
#endif
#endif

#if BUILD_COMPLEX16 == 1
#ifdef ZGEMM3M_DEFAULT_Q
  TABLE_NAME.zgemm3m_q = ZGEMM3M_DEFAULT_Q;
#else
  TABLE_NAME.zgemm3m_q = DGEMM_DEFAULT_Q;
#endif
#endif

#ifdef EXPRECISION
  TABLE_NAME.qgemm_q = QGEMM_DEFAULT_Q;
  TABLE_NAME.xgemm_q = XGEMM_DEFAULT_Q;
  TABLE_NAME.xgemm3m_q = QGEMM_DEFAULT_Q;
#endif

#if defined(CORE_KATMAI)  || defined(CORE_COPPERMINE) || defined(CORE_BANIAS) || defined(CORE_YONAH) || defined(CORE_ATHLON)

#ifdef DEBUG
  fprintf(stderr, "Katmai, Coppermine, Banias, Athlon\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  64 * (l2 >> 7);
#endif
#if BUILD_DOUBLE == 1
  TABLE_NAME.dgemm_p =  32 * (l2 >> 7);
#endif
#if BUILD_COMPLEX==1 
  TABLE_NAME.cgemm_p =  32 * (l2 >> 7);
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  16 * (l2 >> 7);
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  16 * (l2 >> 7);
  TABLE_NAME.xgemm_p =   8 * (l2 >> 7);
#endif
#endif

#ifdef CORE_NORTHWOOD

#ifdef DEBUG
  fprintf(stderr, "Northwood\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  96 * (l2 >> 7);
#endif
#if BUILD_DOUBLE == 1
  TABLE_NAME.dgemm_p =  48 * (l2 >> 7);
#endif
#if BUILD_COMPLEX==1 
  TABLE_NAME.cgemm_p =  48 * (l2 >> 7);
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  24 * (l2 >> 7);
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  24 * (l2 >> 7);
  TABLE_NAME.xgemm_p =  12 * (l2 >> 7);
#endif
#endif

#ifdef ATOM

#ifdef DEBUG
  fprintf(stderr, "Atom\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = 256;
#endif
#if BUILD_DOUBLE ==1
  TABLE_NAME.dgemm_p = 128;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p = 128;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  64;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  64;
  TABLE_NAME.xgemm_p =  32;
#endif
#endif

#ifdef CORE_PRESCOTT

#ifdef DEBUG
  fprintf(stderr, "Prescott\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  56 * (l2 >> 7);
#endif
#if BUILD_DOUBLE ==1 
  TABLE_NAME.dgemm_p =  28 * (l2 >> 7);
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p =  28 * (l2 >> 7);
#endif
#if BUILD_COMPLEX16 == 1
  TABLE_NAME.zgemm_p =  14 * (l2 >> 7);
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  14 * (l2 >> 7);
  TABLE_NAME.xgemm_p =   7 * (l2 >> 7);
#endif
#endif

#ifdef CORE2

#ifdef DEBUG
  fprintf(stderr, "Core2\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  92 * (l2 >> 9) + 8;
#endif
#if BUILD_DOUBLE==1
  TABLE_NAME.dgemm_p =  46 * (l2 >> 9) + 8;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p =  46 * (l2 >> 9) + 4;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  23 * (l2 >> 9) + 4;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  92 * (l2 >> 9) + 8;
  TABLE_NAME.xgemm_p =  46 * (l2 >> 9) + 4;
#endif
#endif

#ifdef PENRYN

#ifdef DEBUG
  fprintf(stderr, "Penryn\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  42 * (l2 >> 9) + 8;
#endif
#if BUILD_DOUBLE == 1
  TABLE_NAME.dgemm_p =  42 * (l2 >> 9) + 8;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p =  21 * (l2 >> 9) + 4;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  21 * (l2 >> 9) + 4;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  42 * (l2 >> 9) + 8;
  TABLE_NAME.xgemm_p =  21 * (l2 >> 9) + 4;
#endif
#endif

#ifdef DUNNINGTON

#ifdef DEBUG
  fprintf(stderr, "Dunnington\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p =  42 * (l2 >> 9) + 8;
#endif
#if BUILD_DOUBLE ==1
  TABLE_NAME.dgemm_p =  42 * (l2 >> 9) + 8;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p =  21 * (l2 >> 9) + 4;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p =  21 * (l2 >> 9) + 4;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  42 * (l2 >> 9) + 8;
  TABLE_NAME.xgemm_p =  21 * (l2 >> 9) + 4;
#endif
#endif


#ifdef NEHALEM

#ifdef DEBUG
  fprintf(stderr, "Nehalem\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef SANDYBRIDGE

#ifdef DEBUG
  fprintf(stderr, "Sandybridge\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef HASWELL

#ifdef DEBUG
  fprintf(stderr, "Haswell\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16)
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#if defined(SKYLAKEX) || defined(COOPERLAKE) || defined(SAPPHIRERAPIDS)

#ifdef DEBUG
  fprintf(stderr, "SkylakeX\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif


#ifdef OPTERON

#ifdef DEBUG
  fprintf(stderr, "Opteron\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = 224 +  56 * (l2 >> 7);
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = 112 +  28 * (l2 >> 7);
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = 112 +  28 * (l2 >> 7);
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p =  56 +  14 * (l2 >> 7);
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p =  56 +  14 * (l2 >> 7);
  TABLE_NAME.xgemm_p =  28 +   7 * (l2 >> 7);
#endif
#endif

#ifdef BARCELONA

#ifdef DEBUG
  fprintf(stderr, "Barcelona\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef BOBCAT

#ifdef DEBUG
  fprintf(stderr, "Bobcate\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef BULLDOZER

#ifdef DEBUG
  fprintf(stderr, "Bulldozer\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef EXCAVATOR

#ifdef DEBUG
  fprintf(stderr, "Excavator\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif


#ifdef PILEDRIVER

#ifdef DEBUG
  fprintf(stderr, "Piledriver\n");
#endif

#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef STEAMROLLER

#ifdef DEBUG
  fprintf(stderr, "Steamroller\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif

#ifdef ZEN

#ifdef DEBUG
  fprintf(stderr, "Zen\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if BUILD_DOUBLE
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if BUILD_COMPLEX16
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif
#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif
#endif


#ifdef NANO

#ifdef DEBUG
  fprintf(stderr, "NANO\n");
#endif

#if  (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
#endif
#if  (BUILD_DOUBLE==1)
  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
#endif
#if (BUILD_COMPLEX==1)
  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
#endif
#if (BUILD_COMPLEX16==1)
  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#endif


#ifdef EXPRECISION
  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
#endif

#endif

#if BUILD_COMPLEX==1
#ifdef CGEMM3M_DEFAULT_P
  TABLE_NAME.cgemm3m_p = CGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.cgemm3m_p = TABLE_NAME.sgemm_p;
#endif
#endif

#if BUILD_COMPLEX16==1
#ifdef ZGEMM3M_DEFAULT_P
  TABLE_NAME.zgemm3m_p = ZGEMM3M_DEFAULT_P;
#else
  TABLE_NAME.zgemm3m_p = TABLE_NAME.dgemm_p;
#endif
#endif

#ifdef EXPRECISION
  TABLE_NAME.xgemm3m_p = TABLE_NAME.qgemm_p;
#endif


#if BUILD_SINGLE == 1
  TABLE_NAME.sgemm_p = ((TABLE_NAME.sgemm_p + SGEMM_DEFAULT_UNROLL_M - 1)/SGEMM_DEFAULT_UNROLL_M) * SGEMM_DEFAULT_UNROLL_M;
#endif
#if BUILD_DOUBLE== 1
  TABLE_NAME.dgemm_p = ((TABLE_NAME.dgemm_p + DGEMM_DEFAULT_UNROLL_M - 1)/DGEMM_DEFAULT_UNROLL_M) * DGEMM_DEFAULT_UNROLL_M;
#endif
#if BUILD_COMPLEX==1
  TABLE_NAME.cgemm_p = ((TABLE_NAME.cgemm_p + CGEMM_DEFAULT_UNROLL_M - 1)/CGEMM_DEFAULT_UNROLL_M) * CGEMM_DEFAULT_UNROLL_M;
#endif
#if BUILD_COMPLEX16==1
  TABLE_NAME.zgemm_p = ((TABLE_NAME.zgemm_p + ZGEMM_DEFAULT_UNROLL_M - 1)/ZGEMM_DEFAULT_UNROLL_M) * ZGEMM_DEFAULT_UNROLL_M;
#endif

#if BUILD_COMPLEX==1
#ifdef CGEMM3M_DEFAULT_UNROLL_M
  TABLE_NAME.cgemm3m_p = ((TABLE_NAME.cgemm3m_p + CGEMM3M_DEFAULT_UNROLL_M - 1)/CGEMM3M_DEFAULT_UNROLL_M) * CGEMM3M_DEFAULT_UNROLL_M;
#else
  TABLE_NAME.cgemm3m_p = ((TABLE_NAME.cgemm3m_p + SGEMM_DEFAULT_UNROLL_M - 1)/SGEMM_DEFAULT_UNROLL_M) * SGEMM_DEFAULT_UNROLL_M;
#endif
#endif

#if BUILD_COMPLEX16==1
#ifdef ZGEMM3M_DEFAULT_UNROLL_M
  TABLE_NAME.zgemm3m_p = ((TABLE_NAME.zgemm3m_p + ZGEMM3M_DEFAULT_UNROLL_M - 1)/ZGEMM3M_DEFAULT_UNROLL_M) * ZGEMM3M_DEFAULT_UNROLL_M;
#else
  TABLE_NAME.zgemm3m_p = ((TABLE_NAME.zgemm3m_p + DGEMM_DEFAULT_UNROLL_M - 1)/DGEMM_DEFAULT_UNROLL_M) * DGEMM_DEFAULT_UNROLL_M;
#endif
#endif

#ifdef QUAD_PRECISION
  TABLE_NAME.qgemm_p = ((TABLE_NAME.qgemm_p + QGEMM_DEFAULT_UNROLL_M - 1)/QGEMM_DEFAULT_UNROLL_M) * QGEMM_DEFAULT_UNROLL_M;
  TABLE_NAME.xgemm_p = ((TABLE_NAME.xgemm_p + XGEMM_DEFAULT_UNROLL_M - 1)/XGEMM_DEFAULT_UNROLL_M) * XGEMM_DEFAULT_UNROLL_M;
  TABLE_NAME.xgemm3m_p = ((TABLE_NAME.xgemm3m_p + QGEMM_DEFAULT_UNROLL_M - 1)/QGEMM_DEFAULT_UNROLL_M) * QGEMM_DEFAULT_UNROLL_M;
#endif

#ifdef DEBUG
  fprintf(stderr, "L2 = %8d DGEMM_P  .. %d\n", l2, TABLE_NAME.dgemm_p);
#endif

#if BUILD_SINGLE==1
  TABLE_NAME.sgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.sgemm_p * TABLE_NAME.sgemm_q *  4 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.sgemm_q *  4) - 15) & ~15);
#endif

#if BUILD_DOUBLE==1
  TABLE_NAME.dgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.dgemm_p * TABLE_NAME.dgemm_q *  8 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.dgemm_q *  8) - 15) & ~15);
#endif

#ifdef EXPRECISION
  TABLE_NAME.qgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.qgemm_p * TABLE_NAME.qgemm_q * 16 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.qgemm_q * 16) - 15) & ~15);
#endif

#if BUILD_COMPLEX ==1 
  TABLE_NAME.cgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.cgemm_p * TABLE_NAME.cgemm_q *  8 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.cgemm_q *  8) - 15) & ~15);
#endif

#if BUILD_COMPLEX16 ==1
  TABLE_NAME.zgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.zgemm_p * TABLE_NAME.zgemm_q * 16 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.zgemm_q * 16) - 15) & ~15);
#endif

#if BUILD_COMPLEX == 1
  TABLE_NAME.cgemm3m_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.cgemm3m_p * TABLE_NAME.cgemm3m_q *  8 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.cgemm3m_q *  8) - 15) & ~15);
#endif

#if BUILD_COMPLEX16 == 1
  TABLE_NAME.zgemm3m_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.zgemm3m_p * TABLE_NAME.zgemm3m_q * 16 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
			       ) / (TABLE_NAME.zgemm3m_q * 16) - 15) & ~15);
#endif



#ifdef EXPRECISION
  TABLE_NAME.xgemm_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.xgemm_p * TABLE_NAME.xgemm_q * 32 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
		       ) / (TABLE_NAME.xgemm_q * 32) - 15) & ~15);

  TABLE_NAME.xgemm3m_r = (((BUFFER_SIZE -
			       ((TABLE_NAME.xgemm3m_p * TABLE_NAME.xgemm3m_q * 32 + TABLE_NAME.offsetA
				 + TABLE_NAME.align) & ~TABLE_NAME.align)
		       ) / (TABLE_NAME.xgemm3m_q * 32) - 15) & ~15);

#endif



}
#endif //POWER
#endif //ZARCH
#endif //(ARCH_MIPS64)
#endif //(ARCH_ARM64)
