# Author: Alfio Lazzaro, alazzaro@cray.com (2014)
#
# This file uses offload mode for the execution of the kernels on the Intel Xeon phi.
# Remember to set the variable target_compile_offload.

#
# target compiler... these are the options used for building the Intel Xeon Phi kernels.
# They should be aggessive enough to e.g. perform vectorization,
# and allow some flexibility in reordering floating point expressions.
# Higher level optimisation (in particular loop nest optimization) should not
# be used.
# Mandatory to add "-offload-attribute-target=mic"
#
target_compile="ftn -O2 -funroll-loops -warn -offload-attribute-target=mic -fpp -finline-functions -nogen-interfaces -openmp"

#
# target compiler used to build the CPU driven code for the offload execution
# of the Intel Xeon Phi kernels.
# You can reuse most of the flags defined in the target_compile variable.
# Replace "-offload-attribute-target=mic" with "-offload=mandatory".
#
target_compile_offload="ftn -O2 -funroll-loops -warn -offload=mandatory -watch=mic-cmd -fpp -finline-functions -nogen-interfaces -openmp"

#
# target dgemm link options... these are the options needed to link blas (e.g. -lblas)
# blas is used as a fall back option for sizes not included in the library or in those cases where it is faster
# the same blas library should thus also be used when libsmm is linked.
#
blas_linking="-mkl=sequential"

#
# SIMD registers type (bytes): sse (16), avx (32), knc (64)
#
SIMD=knc

#
# Number of threads to be used on the card.
# Do not use hyperthreads!
#
MIC_OMP_NUM_THREADS=58

#
# host compiler... this is used only to compile a few tools needed to build
# the library. The library itself is not compiled this way.
# This compiler needs to be able to deal with some Fortran2003 constructs.
#
host_compile="ftn -O2"

#
# Set KMP affinity and other variables
#
export KMP_AFFINITY=verbose,granularity=fine,scatter
export OFFLOAD_REPORT=3
export OMP_STACKSIZE=256M
export MIC_STACKSIZE=256M
export CRAYPE_LINK_TYPE=dynamic

#
# Set the aprun command and its options for batch submission
#
aprun_cmd="aprun -n 1 -N 1 -d ${ntasks} -r 1 -cc none"
