点击次数：

Access to the cluster

ssh username@211.86.151.115

Compilation instructions

Before you start compiling Nektar++, you need to load the following modules

module load boost/1.82.0 cmake/3.19.0 hpcx/2.19/hpcx \
tbb/latest compiler-rt/latest mkl/2022.1.0

Download the source code

git clone https://gitlab.nektar.info/gaoak/nektar.git nektar++
cd nektar++

Put third party softwares into nektar++/ThirdParty

fftw-3.2.2-double.tar.gz  gsmpi-1.2.1_2.tar.bz2  scotch-7.0.1           zlib-1.2.9.tar.gz
fftw-3.2.2-single.tar.gz  hdf5-1.12.3.tar.bz2    scotch_7.0.1.tar.gz
fftw-3.2.2.tar.gz         scotch_6.0.4.tar.gz    tinyxml_2_6_2.tar.bz2and apply the patch

diff --git a/cmake/ThirdPartyScotch.cmake b/cmake/ThirdPartyScotch.cmake

index 54024eaaa..5d33eccf8 100644

--- a/cmake/ThirdPartyScotch.cmake

+++ b/cmake/ThirdPartyScotch.cmake

@@ -70,8 +70,8 @@ IF (NEKTAR_USE_SCOTCH)

         EXTERNALPROJECT_ADD(

             scotch-7.0.1

             PREFIX ${TPSRC}

-            GIT_REPOSITORY https://gitlab.inria.fr/scotch/scotch.git

-            GIT_TAG v7.0.1

+            DOWNLOAD_COMMAND ""

+            UPDATE_COMMAND ""

             STAMP_DIR ${TPBUILD}/stamp

             DOWNLOAD_DIR ${TPSRC}

             SOURCE_DIR ${TPSRC}/scotch-7.0.1

Create the build directory

cd nektar++
mkdir build
cd build

for common build, run cmake

CC=mpicc CXX=mpicxx
cmake -DNEKTAR_USE_MPI=ON -DNEKTAR_USE_MKL=ON \
-DNEKTAR_USE_SYSTEM_BLAS_LAPACK=OFF \
-DNEKTAR_USE_FFTW=ON -DNEKTAR_USE_HDF5=ON \
-DTHIRDPARTY_BUILD_BOOST=OFF \
-DTHIRDPARTY_BUILD_HDF5=ON  \
-DTHIRDPARTY_BUILD_FFTW=ON  \
-DTHIRDPARTY_BUILD_BLAS_LAPACK=OFF   ..

for AVX512 build, run cmake

CC=mpicc CXX=mpicxx
cmake -DNEKTAR_USE_MPI=ON -DNEKTAR_USE_MKL=ON \
-DNEKTAR_USE_SYSTEM_BLAS_LAPACK=OFF \
-DNEKTAR_USE_FFTW=ON -DNEKTAR_USE_HDF5=ON \
-DTHIRDPARTY_BUILD_BOOST=OFF \
-DTHIRDPARTY_BUILD_HDF5=ON  \
-DNEKTAR_ENABLE_SIMD=AVX512  \
-DCMAKE_CXX_FLAGS=-mavx512f\ -mfma  \
-DTHIRDPARTY_BUILD_FFTW=ON  \
-DTHIRDPARTY_BUILD_BLAS_LAPACK=OFF   ..

Then run

make -j4 install

Submit a job using sbatch, sbatch job.sub, with a job.sub file

run module load first

module load boost/1.82.0 cmake/3.19.0 hpcx/2.19/hpcx \
tbb/latest compiler-rt/latest mkl/2022.1.0

then, sbatch job.sh with file

#!/bin/sh
#SBATCH -J 2Dcase
#SBATCH -o job-%j.log
#SBATCH -e job-%j.err
#SBATCH -p CPU-64C256GB
#SBATCH -N 1
#SBATCH -n 64
#SBATCH --exclude=cnode57,cnode58,cnode59,cnode66
#SBATCH --exclusive
#SBATCH --time=4:10:0
#SBATCH --qos=qos_cpu_64c256gb

echo Time is `date`
echo Directory is $PWD
echo This job runs on the following nodes:
echo $SLURM_JOB_NODELIST
echo This job has allocated $SLURM_JOB_CPUS_PER_NODE cpu cores.

export NEK_DIR=/home/ses/agao20/code/nektar/build
export NEK_BUILD=$NEK_DIR/dist/bin
export LD_LIBRARY_PATH=$NEK_DIR/ThirdParty/dist/lib:$NEK_DIR/dist/lib64:$LD_LIBRARY_PATH
export OMP_NUM_THREADS=1

MPIRUN=mpirun #Intel mpi and Open MPI
$MPIRUN --mca pml ob1 \
    --mca btl self,vader,tcp \
    --mca btl_vader_single_copy_mechanism none \
    --mca coll ^ucc,hcoll \ 
    $NEK_BUILD/IncNavierStokesSolver wing.xml wingc.xml  -i Hdf5 -v --set-start-chknumber 0 > runlog 2>&1

References

https://www.nektar.info/nektar-on-archer2/
https://www.nektar.info/nektar-ic-hpc/