Commit 89c4e897 authored by GIORGIO AMATI's avatar GIORGIO AMATI
Browse files
parents 26c42b8a 13a8e641
#!/bin/bash
set -eu
specfem_top=$PWD/../
specfem_bench=$specfem_top/benchmarks
specfem_ref=$specfem_top/references
rundir=./data
specfem_version=git
compiler=intel
use_vectorization=true
use_openmp=false
use_cuda=false
nex_xi=512
nex_eta=512
nproc_xi=64
nproc_eta=64
nodes=64
ranks=4096
ranks_per_node=$[($ranks+$nodes-1)/$nodes]
threads_per_rank=$[64/$ranks_per_node]
echo "ranks_per_node="$ranks_per_node
echo "threads_per_rank="$threads_per_rank
compiler_module=intel/pe-xe-2017--binary
mpi_module=intelmpi/2017--binary
flags='-ftz -assume byterecl -align sequence -std08 -diag-disable 6477 -implicitnone -gen-interfaces -warn all -check nobounds -axMIC-AVX512 -O3'
cflags='-O3'
#cuda_module=cuda/9.2.88
cuda_module=
module load $compiler_module $mpi_module $cuda_module
#module load make/4.2.1
case $compiler in
pgi)
export FC=pgfortran
export CC=pgcc
export CXX=pgc++
export OMP_FCFLAGS=-mp
;;
gnu)
export FC=gfortran
export CC=gcc
export CXX=g++
export OMP_FCFLAGS=-fopenmp
;;
*)
;;
esac
export MPIFC=mpiifort
export FCFLAGS=$flags
export CFLAGS=$cflags
export CXXFLAGS=$cflags
export FLAGS_CHECK=$cflags
export LOCAL_PATH_IS_ALSO_GLOBAL=true
if [ $use_vectorization == true ]; then
if [ $specfem_version == git ]; then
ENABLE_VECTORIZATION=--enable-vectorization
else
ENABLE_VECTORIZATION=--with-vec
fi
else
ENABLE_VECTORIZATION=
fi
if [ $use_openmp == true ]; then
if [ $specfem_version == git ]; then
echo git version doesn\'t benefit from using --enable-openmp
exit 1
ENABLE_OPENMP=--enable-openmp
else
echo pcp version doesn\'t have --enable-openmp
exit 1
fi
else
ENABLE_OPENMP=
fi
if [ $use_cuda == true ]; then
if [ $specfem_version == git ]; then
ENABLE_CUDA=--with-cuda=cuda5
pu=gpu
else
echo pcp version doesn\'t work when using --with-cuda=cuda5
exit 1
ENABLE_CUDA=--with-cuda=cuda5
pu=gpu
fi
else
ENABLE_CUDA=
pu=cpu
fi
#!/bin/bash
set -eu
current_setup=$(realpath $(dirname $BASH_SOURCE))
pushd $current_setup
. 00-settings
pushd $specfem_top/$specfem_version
if [ $specfem_version == pcp ]; then
sed -i '/\<ppc64\>/{s;\(ppc64\);\1le;}' config.guess
sed -i '/^\s\+pgf95/{s;\(pgf95\);pgfortran|\*\/pgfortran|\1;}' flags.guess
sed -i '/no-gpu version\\/{s;\\;;}' src/specfem3D/rules.mk
if [ $use_cuda == true ]; then
sed -i '/\!call compute_kernels_strgth_noise_gpu/{s;\!;;}' src/specfem3D/noise_tomography.f90
else
sed -i '/\scall compute_kernels_strgth_noise_gpu/{s;\(call\);\!\1;}' src/specfem3D/noise_tomography.f90
fi
fi
mkdir -p $current_setup/logs
./configure $ENABLE_VECTORIZATION $ENABLE_OPENMP $ENABLE_CUDA |& tee $current_setup/logs/config_summary.log
mv config.log $current_setup/logs/
popd
popd
#!/bin/bash
set -eu
current_setup=$(realpath $(dirname $BASH_SOURCE))
pushd $current_setup
. 00-settings
sed -i "/NEX_XI\s*=/{s;.*;NEX_XI = $nex_xi;}" DATA/Par_file
sed -i "/NEX_ETA\s*=/{s;.*;NEX_ETA = $nex_eta;}" DATA/Par_file
sed -i "/NPROC_XI\s*=/{s;.*;NPROC_XI = $nproc_xi;}" DATA/Par_file
sed -i "/NPROC_ETA\s*=/{s;.*;NPROC_ETA = $nproc_eta;}" DATA/Par_file
if [ $specfem_version == git ]; then
sed -i '/NT_DUMP_ATTENUATION\s*=/{s;\(.*\);\1\nMEMORY_INSTALLED_PER_CORE_IN_GB = 80.0d0\nPERCENT_OF_MEM_TO_USE_PER_CORE = 90.0d0;}' DATA/Par_file
sed -i '/NTSTEP_BETWEEN_READ_ADJSRC\s*=/{s;\(.*\);\1\nUSE_FORCE_POINT_SOURCE = .false.;}' DATA/Par_file
sed -i '/OUTPUT_SEISMOS_SAC_BINARY\s*=/{s;\(.*\);\1\nOUTPUT_SEISMOS_ASDF = .false.;}' DATA/Par_file
sed -i '/PRINT_SOURCE_TIME_FUNCTION\s*=/{s;\(.*\);\1\nREAD_ADJSRC_ASDF = .false.;}' DATA/Par_file
sed -i '/SAVE_REGULAR_KL\s*=/{s;\(.*\);\1\nNUMBER_OF_SIMULTANEOUS_RUNS = 1\nBROADCAST_SAME_MESH_AND_MODEL = .false.;}' DATA/Par_file
sed -i '/ADIOS_FOR_AVS_DX\s*=/{s;\(.*\);\1\nADIOS_FOR_KERNELS = .true.\nADIOS_FOR_MODELS = .true.\nADIOS_FOR_UNDO_ATTENUATION = .true.;}' DATA/Par_file
sed -i '/ADIOS_FOR_AVS_DX\s*=/{s;\(.*\);\1\nSAVE_SEISMOGRAMS_STRAIN = .false.\nADIOS_FOR_MODELS = .true.\nSAVE_SEISMOGRAMS_IN_ADJOINT_RUN = .false.;}' DATA/Par_file
fi
if [ $use_cuda == true ]; then
sed -i '/GPU_MODE\s*=/{s;.*;GPU_MODE = .true.;}' DATA/Par_file
else
sed -i '/GPU_MODE\s*=/{s;.*;GPU_MODE = .false.;}' DATA/Par_file
fi
pushd $specfem_top/$specfem_version
rsync -azP $current_setup/DATA/Par_file DATA/
make clean
make xcreate_header_file |& tee $current_setup/logs/make.xcreate_header_file.log
make xmeshfem3D |& tee $current_setup/logs/make.xmeshfem3D.log
make xspecfem3D |& tee $current_setup/logs/make.xspecfem3D.log
mkdir -p $current_setup/OUTPUT_FILES
rsync -azP setup/* $current_setup/OUTPUT_FILES/
rsync -azP OUTPUT_FILES/values_from_mesher.h $current_setup/OUTPUT_FILES/
rsync -azP bin $current_setup/
for dir in crust2.0 s362ani QRFSI12 topo_bathy; do
rsync -azP DATA/$dir $specfem_bench/DATA/$specfem_version/
[ -L $current_setup/DATA/$dir ] && rm $current_setup/DATA/$dir
ln -sfrn $specfem_bench/DATA/$specfem_version/$dir $current_setup/DATA/$dir
done
popd
mkdir -p DATABASES_MPI
popd
#!/bin/bash
set -eu
current_setup=$(realpath $(dirname $BASH_SOURCE))
pushd $current_setup
. 00-settings
echo "number of nodes=" $nodes
echo "number of total ranks=" $ranks
echo "number of ranks per node=" $ranks_per_node
sbatch --nodes=$nodes --ntasks=$ranks --ntasks-per-node=$ranks_per_node --exclusive 04-submit
popd
#!/bin/bash
#SBATCH --partition=knl_usr_prod
#SBATCH --time=06:00:00
#SBATCH --account=CHEPR_devel_0
set -eu
current_setup=$SLURM_SUBMIT_DIR
pushd $current_setup
. 00-settings
local_setup=$rundir/$(basename $current_setup)
local_setup=$current_setup
#for host in $(scontrol show hostname $SLURM_JOB_NODELIST); do
# rsync -azPL --exclude 'DATABASES_MPI/proc*' --exclude 'OUTPUT_FILES/*.ascii' --exclude 'slurm-*' ./* $host:/$local_setup/
#done
pushd $local_setup
export OMP_NUM_THREADS=$threads_per_rank
export SLURM_TASKS_PER_NODE="$ranks_per_node(x$nodes)"
srun --cpu_bind=core ./bin/xmeshfem3D
srun --cpu_bind=core ./bin/xspecfem3D
popd
#for host in $(scontrol show hostname $SLURM_JOB_NODELIST); do
# rsync -azPK --remove-source-files $host:$local_setup/* ./
# rsync -azPK --remove-source-files --exclude 'DATABASES_MPI/proc*' $host:$local_setup/* ./
# ssh $host "rm -rf $local_setup"
#done
popd
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment